
# 导入包
import pandas as pd
import numpy as np
from faker import Faker


# 伪造数据用于示例
faker = Faker(locale='zh-CN')
Faker.seed(12)
raw_df = pd.DataFrame({
    'name': [faker.name() for x in range(5)],
    'gender': [faker.profile()['sex'] for x in range(5)],
    'age': np.random.randint(18, 80, 5),
    'date': [faker.date() for x in range(5)],
    'amount': np.random.randint(10, 10000, 5),
    'postcode': [faker.postcode() for x in range(5)],
    'cnt': [np.NaN, 10, np.NaN, 20, np.NaN]
})
raw_df


# 先查看下数据集基本信息
raw_df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 5 entries, 0 to 4
Data columns (total 7 columns):
 #   Column    Non-Null Count  Dtype  
---  ------    --------------  -----  
 0   name      5 non-null      object 
 1   gender    5 non-null      object 
 2   age       5 non-null      int32  
 3   date      5 non-null      object 
 4   amount    5 non-null      int32  
 5   postcode  5 non-null      object 
 6   cnt       2 non-null      float64
dtypes: float64(1), int32(2), object(4)
memory usage: 368.0+ bytes


df = raw_df.copy()
df = df.convert_dtypes()
df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 5 entries, 0 to 4
Data columns (total 7 columns):
 #   Column    Non-Null Count  Dtype 
---  ------    --------------  ----- 
 0   name      5 non-null      string
 1   gender    5 non-null      string
 2   age       5 non-null      Int32 
 3   date      5 non-null      string
 4   amount    5 non-null      Int32 
 5   postcode  5 non-null      string
 6   cnt       2 non-null      Int64 
dtypes: Int32(2), Int64(1), string(4)
memory usage: 383.0 bytes


df = raw_df.copy()
df['is_male'] = df['gender'] == 'M'
df['is_male'] = df['is_male'].astype('int8')  # 将布尔值True、False转成数值1、0
df = df.astype({
    'name': 'string',
    'gender': 'string',
    'date': 'datetime64[ns]'
})
df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 5 entries, 0 to 4
Data columns (total 8 columns):
 #   Column    Non-Null Count  Dtype         
---  ------    --------------  -----         
 0   name      5 non-null      string        
 1   gender    5 non-null      string        
 2   age       5 non-null      int32         
 3   date      5 non-null      datetime64[ns]
 4   amount    5 non-null      int32         
 5   postcode  5 non-null      object        
 6   cnt       2 non-null      float64       
 7   is_male   5 non-null      int8          
dtypes: datetime64[ns](1), float64(1), int32(2), int8(1), object(1), string(2)
memory usage: 373.0+ bytes


df = raw_df.copy()
df.select_dtypes(include=np.number)  # 选择所有数值列
df.select_dtypes(exclude='object')  # 选择所有非object类型的列


df = raw_df.copy()
df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True)
df.dtypes

name                object
gender              object
age                  int32
date        datetime64[ns]
amount               int32
postcode            object
cnt                float64
dtype: object


df = raw_df.copy()
df['postcode'] = pd.to_numeric(df['postcode'])
df.dtypes

name         object
gender       object
age           int32
date         object
amount        int32
postcode      int64
cnt         float64
dtype: object


df = raw_df.copy()
df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True)
df['week'] = df['date'].dt.to_period('W').dt.start_time
df['month'] = df['date'].dt.to_period('M')
df['month_start'] = df['date'].dt.to_period('M').dt.start_time
df['month_end'] = df['date'].dt.to_period('M').dt.end_time
df['year'] = df['date'].dt.to_period('Y')
df['year_start'] = df['date'].dt.to_period('Y').dt.start_time
df['year_end'] = df['date'].dt.to_period('Y').dt.end_time
df[['date', 'week', 'month', 'month_start', 'month_end', 'year', 'year_start', 'year_end']]


df = raw_df.copy()
df['is_good'] = df['name'].isin(['吴柳', '罗峰']).astype('int8')
df


df = raw_df.copy()
df['amount_5k_1w'] = df['amount'].between(5000, 10000, inclusive='left')
df


df = raw_df.copy()
df.nlargest(3, 'amount')


df = raw_df.copy()
bins = [0, 30, 60, 100]
labels = ['青', '中', '老']
df['age_group'] = pd.cut(df['age'], bins, labels=labels)
df


df = raw_df.copy()
df['amount_mean'] = df.groupby('gender')['amount'].transform('mean')
df


df = raw_df.copy()
df['gender_new'] = df['gender'].map({'M': 1, 'F': 0})
df['amount_new'] = df['amount'].map(lambda x: round(x/1000, 2))
df['amount_grp'] = df['amount'].map(lambda x: 'low' if x < 5000 else 'high')
df


df = raw_df.copy()
def add_value(x, value):
    return x + value
df['age_new'] = df['age'].apply(add_value, value=1)
df


df = raw_df.copy()
grouped = df.groupby(by=['gender']).agg({'amount': [('amount_max', 'max'), ('amount_min', 'min')], 'age': [('age_median', 'median'), ('age_min','min')], 'name': [('cnt', 'nunique')]})
grouped.columns = grouped.columns.droplevel()
grouped


df = raw_df.copy()
df.pivot_table(values=['amount'], index=['date'], columns=['gender'], aggfunc='sum')


df = raw_df.copy()
df['amount_cumsum'] = df['amount'].cumsum()
df


df = raw_df.copy()
df['amount_cumsum'] = df.groupby(by=['gender'])['amount'].cumsum()
df


df = raw_df.copy()
df['amount_pct_change'] = df['amount'].pct_change()
df


df = raw_df.copy()
df['gender'].nunique()

2


df = raw_df.copy()
df.corr()

	name	gender	age	date	amount	postcode	cnt
0	刘倩	F	18	2012-04-06	1352	855589	NaN
1	吴柳	F	31	2009-04-11	1152	629635	10.0
2	陈桂芳	M	62	1988-10-18	280	124556	NaN
3	罗峰	F	34	1995-10-02	4154	381814	20.0
4	黄雪	M	78	1992-11-05	7501	514273	NaN

	age	amount	cnt
0	18	1352	NaN
1	31	1152	10.0
2	62	280	NaN
3	34	4154	20.0
4	78	7501	NaN

	date	week	month	month_start	month_end	year	year_start	year_end
0	2012-04-06	2012-04-02	2012-04	2012-04-01	2012-04-30 23:59:59.999999999	2012	2012-01-01	2012-12-31 23:59:59.999999999
1	2009-04-11	2009-04-06	2009-04	2009-04-01	2009-04-30 23:59:59.999999999	2009	2009-01-01	2009-12-31 23:59:59.999999999
2	1988-10-18	1988-10-17	1988-10	1988-10-01	1988-10-31 23:59:59.999999999	1988	1988-01-01	1988-12-31 23:59:59.999999999
3	1995-10-02	1995-10-02	1995-10	1995-10-01	1995-10-31 23:59:59.999999999	1995	1995-01-01	1995-12-31 23:59:59.999999999
4	1992-11-05	1992-11-02	1992-11	1992-11-01	1992-11-30 23:59:59.999999999	1992	1992-01-01	1992-12-31 23:59:59.999999999

	name	gender	age	date	amount	postcode	cnt	is_good
0	刘倩	F	18	2012-04-06	1352	855589	NaN	0
1	吴柳	F	31	2009-04-11	1152	629635	10.0	1
2	陈桂芳	M	62	1988-10-18	280	124556	NaN	0
3	罗峰	F	34	1995-10-02	4154	381814	20.0	1
4	黄雪	M	78	1992-11-05	7501	514273	NaN	0

	name	gender	age	date	amount	postcode	cnt	amount_5k_1w
0	刘倩	F	18	2012-04-06	1352	855589	NaN	False
1	吴柳	F	31	2009-04-11	1152	629635	10.0	False
2	陈桂芳	M	62	1988-10-18	280	124556	NaN	False
3	罗峰	F	34	1995-10-02	4154	381814	20.0	False
4	黄雪	M	78	1992-11-05	7501	514273	NaN	True

pandas一些高效好用的函数方法¶

智能转换数据类型：¶

转换数据类型：¶

按数据类型一次性选择所有目标列：¶

目标值转成日期时间值：¶

数值字符串转成数值类型：¶

日期时间的推移计算：¶

判断是否包含枚举值：¶

区间比较判断：¶

排序取前n行：¶

分组：¶

分组统计后合并到原始数据中：¶

将 Series 中的每个值替换为另一个值：¶

对 Series 的值应用函数：¶

分组统计：¶

透视表：¶

累计和：¶

百分比变化或环比同比：¶

去重计数：¶

相关系数：¶

	age	amount	cnt
age	1.000000	0.548072	1.0
amount	0.548072	1.000000	1.0
cnt	1.000000	1.000000	1.0

	amount_max	amount_min	age_median	age_min	cnt
gender
F	4154	1152	31.0	18	3
M	7501	280	70.0	62	2