我的Python分析成长之路11

 1 import numpy as np
 2 s1 = pd.Series([1,2],index=["a","b"])
 3 s2 = pd.Series([3,4,5],index=["c",'d',"e"])
 4 s3 = pd.Series([6,7],index=["f","g"])
 5 s4 = pd.concat([s1,s2,s3])  #默认按列连接
 6 print(s4)
 7 s5 = pd.concat([s1,s2,s3],axis=1)  #按横向连接
 8 print(s5)   #没有重叠部分用缺失值填充
 9 print(pd.concat([s1,s4],axis=1,join="inner"))   #值保留重叠部分
10 df1= pd.DataFrame(np.arange(6).reshape(3,2),index=["a","b","c"],columns=["one","two"])
11 df2 = pd.DataFrame(5+np.arange(4).reshape(2,2),index=["a","c"],columns=["three","four"])
12 print(pd.concat([df1,df2],axis=1,keys=["level1","level2"]))
13 print(pd.concat({"level1":df1,"level2":df2},axis=1))

1 df1= pd.DataFrame(np.arange(6).reshape(3,2),index=["a","b","c"],columns=["one","two"])
2 df2 = pd.DataFrame(5+np.arange(4).reshape(2,2),index=["a","c"],columns=["one","two"])
3 print(df1.append(df2))

1 pd.DataFrame({"key":["b",'b',"a","c"],"data1":np.arange(4)})
2 df2 = pd.DataFrame({"key":["a","b","d"],"data2":np.arange(3)})
3 print(pd.merge(df1,df2,on="key"))    #主键必须相同，且保留公共部分
4 print(pd.merge(df1,df2,how='outer'))  #用nan填充
5 df3 = pd.DataFrame({"key1":["b",'b',"a","c"],"data1":np.arange(4)})
6 df4 = pd.DataFrame({"key2":["a","b","d"],"data2":np.arange(3)})
7 print(pd.merge(df3,df4,left_on="key1",right_on="key2")) #如果两个主键不同就分别接受主键
8 print(pd.merge(df3,df4,left_on="key1",right_on="key2",how="outer")) 

1 df3 = pd.DataFrame({"key1":[np.nan,'b',"a","c"],"data1":[1,np.nan,2,np.nan]})
2 df4 = pd.DataFrame({"key1":["a",np.nan,"d",np.nan],"data1":[np.nan,1,np.nan,3]})
3 print(df3.combine_first(df4))  #用df4中的值重叠填充df3中的缺失值

 1 import pandas as pd
 2 lists=[1,2,3,2,5]
 3 print(pd.Series(lists).drop_duplicates())  #利用pandas.drop_duplicates()去重
 4 print(set(lists))  #利用set去重
 5 
 6 def req(lists):   
 7     #定义去重函数，利用list去重
 8     new_lists =[]
 9     for i in lists:
10         if i not in new_lists:
11             new_lists.append(i)
12     return new_lists
13 print(req(lists))

def FeatureEqual(df):
    dfequal = pd.DataFrame([],index=df.columns,columns=df.columns)
    for i in df.columns:
        for j in df.columns:
            dfequal.loc[i:j]=dfequal[:,i].equals(dfequal[:,j])
    return dfequal
dfequal = FeatureEqual()   #调用函数得特征重复的值
l = dfequal.shape[0]
list2 =[]  #记录需要去重的列
for m in range(l):
    for n in range(m+1,l):
        if dfequal.iloc[m:n] and (dfequal.columns[n] not in list2):
            list2.append(dfequal.columns[n])
print("需要去重的列:",list2)
df.drop(list2,axis=1,inplace=True)   #然后去重

1 import pandas as pd
2 import numpy as np
3 df = pd.DataFrame({"one":[1,2,3,np.nan],"two":[np.nan,4,5,6],"three":[10,9,7,8]})
4 print(df)
5 print(df.dropna(axis=0))#删除缺失行
6 print(df.dropna(axis=1))  #删除缺失列
7 print(df.dropna(axis=0,how="all"))  #删除全为缺失值的行
8 print(df.dropna(axis=0,thresh=2))  #保留含有2个即以上非缺失值的行

print(df.fillna(method="ffill",axis=0))  #利用上一个非缺失值填充缺失值
print(df.fillna(df.mean(),axis=0))  #利用列向的均值来填充缺失值
from sklearn.preprocessing import Imputer   #利用Impute类来实现
imr = Imputer(strategy="median",axis=1)
df = imr.fit_transform(df.values)
print(df)
print(type(df))

 1 import numpy as np
 2 import scipy.interpolate as sci
 3 x = np.array([1,2,3,4,7,8,9])
 4 y1 = np.array([2,8,18,32,50,128,162])
 5 y2 = np.array([3,5,7,9,11,17,19])
 6 linearvalue1 = sci.interp1d(x,y1,kind="linear")
 7 linearvalue2 = sci.interp1d(x,y2,kind="linear")
 8 print("当x为6、7时，使用线性插值y1为：",linearvalue1([5,6]))
 9 print("当x为6、7时，使用线性插值y2为:",linearvalue2([5,6]))
10 linearvalue3 = sci.lagrange(x,y1)
11 linearvalue4 = sci.lagrange(x,y2)
12 print("当x为6、7时，使用拉格朗日插值y1为：",linearvalue3([5,6]))
13 print("当x为6、7时，使用拉格朗日插值y2为:",linearvalue4([5,6]))
14 linearvalue5 = sci.spline(x,y1,xnew=np.array([5,6]))
15 linearvalue6 = sci.spline(x,y2,xnew=np.array([5,6]))
16 print("当x为5、6时，使用样条插值",linearvalue5)
17 print("当x为5、5时，使用样条插值",linearvalue6)

 1 def Outrange(serl):
 2     outbool = (serl.mean()-3*serl.std()>serl)|(serl.mean()+3*serl.std()<serl)
 3     index = np.arange(serl.shape[0])
 4     outrange = serl.iloc[index][outbool]
 5     return outrange
 6 Outrange(serl)  #调用 获得异常
 7 import matplotlib.pyplot as plt   
 8 plt.figure(figsize=(10,8))
 9 p = plt.boxplot(x)   #使用箱型图来检测异常值
10 outrange = p['fliers'][0].get_ydata()  #获得异常值

 1 def MinMaxScale(x):   #离差标准化
 2     x = (x-x.min())/(x.max()-x.min())
 3     return x
 4 from sklearn.preprocessing import MinMaxScaler #机器学习库中调用 但是为
 5 mms = MinMaxScaler()
 6 mms.fit_transform(X)
 7 def Standscale(x):
 8     x = (x-x.mean())/x.std()
 9     return x
10 std = Standscale(x)
11 from sklearn.preprocessing import StandardScaler
12 std = StandardScaler()
13 std.fit_transform(x)
14 def decimalscale(x):  #小数位标准化数据
15     x = x/10**np.ceil(np.log(x.abs().max()))
16     return x
17 decimalscale(x)

import pandas as pd
from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()
x = le.fit_transform(x)
from sklearn.preprocessing import OneHotEncoder
ohe = OneHotEncoder()
ohe.fit_transform(x)
pd.get_dummies(x)

def SamRateCut(data,k):  #自定义等频法
    w = data.quantile(0.1+1/k,1/k)
    data = pd.cut(data,w)
    return data
SamRateCut(data,k) #调用