# 导入基本库 import numpy as np import pandas as pd
import os os.getcwd()
'C:\\Users\\Hello\\Desktop\\hands-on-data-analysis-master\\chapterTwo'
# 载入data文件中的:train-left-up.csv df=pd.read_csv('./data/train-left-up.csv') df.head()
PassengerId | Survived | Pclass | Name | |
---|---|---|---|---|
0 | 1 | 0 | 3 | Braund, Mr. Owen Harris |
1 | 2 | 1 | 1 | Cumings, Mrs. John Bradley (Florence Briggs Th... |
2 | 3 | 1 | 3 | Heikkinen, Miss. Laina |
3 | 4 | 1 | 1 | Futrelle, Mrs. Jacques Heath (Lily May Peel) |
4 | 5 | 0 | 3 | Allen, Mr. William Henry |
#写入代码 text_left_up = pd.read_csv("./data/train-left-up.csv") text_left_down = pd.read_csv("data/train-left-down.csv") text_right_up = pd.read_csv("data/train-right-up.csv") text_right_down = pd.read_csv("data/train-right-down.csv")
#写入代码 text_left_up.head(2)
PassengerId | Survived | Pclass | Name | |
---|---|---|---|---|
0 | 1 | 0 | 3 | Braund, Mr. Owen Harris |
1 | 2 | 1 | 1 | Cumings, Mrs. John Bradley (Florence Briggs Th... |
text_left_down.head(2)
PassengerId | Survived | Pclass | Name | |
---|---|---|---|---|
0 | 440 | 0 | 2 | Kvillner, Mr. Johan Henrik Johannesson |
1 | 441 | 1 | 2 | Hart, Mrs. Benjamin (Esther Ada Bloomfield) |
import torch #!torch.cat()/? #cat=torch.cat(torch.tensor(np.array(text_left_up,text_left_down)),dim=0) #cat
#dataframe=pd.DataFrame(text_left_up,text_left_down,text_right_up,text_right_down).values text_right_up.head(2)
Sex | Age | SibSp | Parch | Ticket | Fare | Cabin | Embarked | |
---|---|---|---|---|---|---|---|---|
0 | male | 22.0 | 1 | 0 | A/5 21171 | 7.2500 | NaN | S |
1 | female | 38.0 | 1 | 0 | PC 17599 | 71.2833 | C85 | C |
text_right_down.head(2)
Sex | Age | SibSp | Parch | Ticket | Fare | Cabin | Embarked | |
---|---|---|---|---|---|---|---|---|
0 | male | 31.0 | 0 | 0 | C.A. 18723 | 10.50 | NaN | S |
1 | female | 45.0 | 1 | 1 | F.C.C. 13529 | 26.25 | NaN | S |
【提示】结合之前我们加载的train.csv数据,大致预测一下上面的数据是什么
上面的数据应该是根据性别、年龄、票价等预测出如果失事后,是否可以存活
#写入代码 list1= [text_left_up,text_right_up] result_up = pd.concat(list1,axis=1) result_up.head(3)
PassengerId | Survived | Pclass | Name | Sex | Age | SibSp | Parch | Ticket | Fare | Cabin | Embarked | |
---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | 1 | 0 | 3 | Braund, Mr. Owen Harris | male | 22.0 | 1 | 0 | A/5 21171 | 7.2500 | NaN | S |
1 | 2 | 1 | 1 | Cumings, Mrs. John Bradley (Florence Briggs Th... | female | 38.0 | 1 | 0 | PC 17599 | 71.2833 | C85 | C |
2 | 3 | 1 | 3 | Heikkinen, Miss. Laina | female | 26.0 | 0 | 0 | STON/O2. 3101282 | 7.9250 | NaN | S |
#写入代码 list2=[text_left_down,text_right_down] result_down=pd.concat(list2,axis=1) list3=[result_up,result_down] result=pd.concat(list3,axis=0)#按行拼接 result.head()
PassengerId | Survived | Pclass | Name | Sex | Age | SibSp | Parch | Ticket | Fare | Cabin | Embarked | |
---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | 1 | 0 | 3 | Braund, Mr. Owen Harris | male | 22.0 | 1 | 0 | A/5 21171 | 7.2500 | NaN | S |
1 | 2 | 1 | 1 | Cumings, Mrs. John Bradley (Florence Briggs Th... | female | 38.0 | 1 | 0 | PC 17599 | 71.2833 | C85 | C |
2 | 3 | 1 | 3 | Heikkinen, Miss. Laina | female | 26.0 | 0 | 0 | STON/O2. 3101282 | 7.9250 | NaN | S |
3 | 4 | 1 | 1 | Futrelle, Mrs. Jacques Heath (Lily May Peel) | female | 35.0 | 1 | 0 | 113803 | 53.1000 | C123 | S |
4 | 5 | 0 | 3 | Allen, Mr. William Henry | male | 35.0 | 0 | 0 | 373450 | 8.0500 | NaN | S |
pandas.DataFrame.join join()是向右扩展添加 DataFrame.join(other, on=None, how=’left’, lsuffix=”, rsuffix=”, sort=False) 通过索引或者指定的列连接两个DataFrame。通过一个list可以一次高效的连接多个DataFrame。 参数说明 other:【DataFrame,或者带有名字的Series,或者DataFrame的list】如果传递的是Series,那么其name属性应当是一个集合, 并且该集合将会作为结果DataFrame的列名 on:【列名称,或者列名称的list/tuple,或者类似形状的数组】连接的列,默认使用索引连接 how:【{‘left’, ‘right’, ‘outer’, ‘inner’}, default: ‘left’】连接的方式,默认为左连接 lsuffix:【string】左DataFrame中重复列的后缀 rsuffix:【string】右DataFrame中重复列的后缀 sort:【boolean, default False】按照字典顺序对结果在连接键上排序。如果为False,连接键的顺序取决于连接类型(关键字)。
#写入代码 result_up = text_left_up.join(text_right_up) result_down = text_left_down.join(text_right_down) result = result_up.append(result_down) result.head()
PassengerId | Survived | Pclass | Name | Sex | Age | SibSp | Parch | Ticket | Fare | Cabin | Embarked | |
---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | 1 | 0 | 3 | Braund, Mr. Owen Harris | male | 22.0 | 1 | 0 | A/5 21171 | 7.2500 | NaN | S |
1 | 2 | 1 | 1 | Cumings, Mrs. John Bradley (Florence Briggs Th... | female | 38.0 | 1 | 0 | PC 17599 | 71.2833 | C85 | C |
2 | 3 | 1 | 3 | Heikkinen, Miss. Laina | female | 26.0 | 0 | 0 | STON/O2. 3101282 | 7.9250 | NaN | S |
3 | 4 | 1 | 1 | Futrelle, Mrs. Jacques Heath (Lily May Peel) | female | 35.0 | 1 | 0 | 113803 | 53.1000 | C123 | S |
4 | 5 | 0 | 3 | Allen, Mr. William Henry | male | 35.0 | 0 | 0 | 373450 | 8.0500 | NaN | S |
#写入代码 result_up = pd.merge(text_left_up,text_right_up,left_index=True,right_index=True) result_down = pd.merge(text_left_down,text_right_down,left_index=True,right_index=True) result = result_up.append(result_down) result.head(2)
PassengerId | Survived | Pclass | Name | Sex | Age | SibSp | Parch | Ticket | Fare | Cabin | Embarked | |
---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | 1 | 0 | 3 | Braund, Mr. Owen Harris | male | 22.0 | 1 | 0 | A/5 21171 | 7.2500 | NaN | S |
1 | 2 | 1 | 1 | Cumings, Mrs. John Bradley (Florence Briggs Th... | female | 38.0 | 1 | 0 | PC 17599 | 71.2833 | C85 | C |
【思考】对比merge、join以及concat的方法的不同以及相同。思考一下在任务四和任务五的情况下,为什么都要求使用DataFrame的append方法,如何只要求使用merge或者join可不可以完成任务四和任务五呢?
对比merge、join以及concat的方法的不同以及相同: join()和merge()一样,支持how的四种模式:inner, left, right, outer,其实join()就是merge()的一种封装,后台调用的正是merge()。 只是为了调用更为简单,才有了join(),merge()和join()是横向拼接,缺省以左侧表格为主,以NaN填充补位 concat是Pandas的方法,缺省是纵向拼接,这一点就和merge,join不同。虽然concat可以指定轴向axis=1来实现横向拼接。 concat定位于数据的连接,这更多的停留在物理融合的层面,而merge则更深入地通过共同的index或是共同项,将两组数据从业务层面进行拼和。 这三种方法各自的特点: merge() merge()函数用于将DataFrame与其他数据以内部联接inner,外部联接outer,左联接left,右联接right的模式进行合并。 以索引或共同列进行拼接。 如果使用共同列进行拼接,则索引将被忽略。 如果按索引进行合并,则索引将被合并为一个唯一索引。 join() join()用于横向连接两个或更多个DataFrames。 后台调用用的是merge,默认为按索引连接。 按照索引或共同列进行拼接。 默认为左连接,可以像merge一样指定参数为右连接,内连接和外连接。 concat() 缺省为垂直连接两个或更多DataFrame和Series。 通过指定axis参数可以实现横向连接。 默认为外连接outer,可以通过join='inner'进行内连接操作。 在任务四和任务五的情况下,使用DataFrame的append方法是为了纵向拼接, 只要求使用merge或者join目前我觉得不可以完成任务四和任务五,因为他们都是横向拼接的
#写入代码 result.to_csv('result.csv')
#写入代码 unit_result=result.stack().head() unit_result.head()
0 PassengerId 1 Survived 0 Pclass 3 Name Braund, Mr. Owen Harris Sex male dtype: object
#写入代码 unit_result.to_csv('unit_result.csv')
将我们的数据变为Series类型的数据 stack()即“堆叠”,作用是将列旋转到行数据
text = pd.read_csv(‘result.csv’)
text.head()
<div> <style> .dataframe thead tr:only-child th { text-align: right; } .dataframe thead th { text-align: left; } .dataframe tbody tr th { vertical-align: top; } </style> <table border="1" class="dataframe"> <thead> <tr style="text-align: right;"> <th></th> <th>Unnamed: 0</th> <th>PassengerId</th> <th>Survived</th> <th>Pclass</th> <th>Name</th> <th>Sex</th> <th>Age</th> <th>SibSp</th> <th>Parch</th> <th>Ticket</th> <th>Fare</th> <th>Cabin</th> <th>Embarked</th> </tr> </thead> <tbody> <tr> <th>0</th> <td>0</td> <td>1</td> <td>0</td> <td>3</td> <td>Braund, Mr. Owen Harris</td> <td>male</td> <td>22.0</td> <td>1</td> <td>0</td> <td>A/5 21171</td> <td>7.2500</td> <td>NaN</td> <td>S</td> </tr> <tr> <th>1</th> <td>1</td> <td>2</td> <td>1</td> <td>1</td> <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td> <td>female</td> <td>38.0</td> <td>1</td> <td>0</td> <td>PC 17599</td> <td>71.2833</td> <td>C85</td> <td>C</td> </tr> <tr> <th>2</th> <td>2</td> <td>3</td> <td>1</td> <td>3</td> <td>Heikkinen, Miss. Laina</td> <td>female</td> <td>26.0</td> <td>0</td> <td>0</td> <td>STON/O2. 3101282</td> <td>7.9250</td> <td>NaN</td> <td>S</td> </tr> <tr> <th>3</th> <td>3</td> <td>4</td> <td>1</td> <td>1</td> <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td> <td>female</td> <td>35.0</td> <td>1</td> <td>0</td> <td>113803</td> <td>53.1000</td> <td>C123</td> <td>S</td> </tr> <tr> <th>4</th> <td>4</td> <td>5</td> <td>0</td> <td>3</td> <td>Allen, Mr. William Henry</td> <td>male</td> <td>35.0</td> <td>0</td> <td>0</td> <td>373450</td> <td>8.0500</td> <td>NaN</td> <td>S</td> </tr> </tbody> </table> </div> # 2 第二章:数据重构 ## 第一部分:数据聚合与运算 ### 2.6 数据运用 #### 2.6.1 任务一:通过教材《Python for Data Analysis》P303、Google or anything来学习了解GroupBy机制 #写入心得 groupby()用于分组计算,最基本的就是组内计数, 求和, 求均值, 求方差 #### 2.4.2:任务二:计算泰坦尼克号男性与女性的平均票价 ```python # 写入代码 mean_ave = text['Fare'].groupby(text['Sex']).mean() mean_ave
Sex female 44.479818 male 25.523893 Name: Fare, dtype: float64
在了解GroupBy机制之后,运用这个机制完成一系列的操作,来达到我们的目的。
下面通过几个任务来熟悉GroupBy机制。
# 写入代码 survived_sex = text['Survived'].groupby(text['Sex']).sum() survived_sex.head()
Sex female 233 male 109 Name: Survived, dtype: int64
# 写入代码 survived_pclass = text['Survived'].groupby(text['Pclass']).sum() survived_pclass
Pclass 1 136 2 87 3 119 Name: Survived, dtype: int64
【提示:】表中的存活那一栏,可以发现如果还活着记为1,死亡记为0
【思考】从数据分析的角度,上面的统计结果可以得出那些结论
#思考心得
存活人数中,一等舱较多,票价较高,且多为女性
【思考】从任务二到任务三中,这些运算可以通过agg()函数来同时计算。并且可以使用rename函数修改列名。你可以按照提示写出这个过程吗?
#思考心得 text.groupby('Sex').agg({'Fare': 'mean', 'Pclass': 'count'}).rename( columns= {'Fare': 'mean_fare', 'Pclass': 'count_pclass'})
mean_fare | count_pclass | |
---|---|---|
Sex | ||
female | 44.479818 | 314 |
male | 25.523893 | 577 |
# 写入代码 text.groupby(['Pclass','Age'])['Fare'].mean()
Pclass Age 1 0.92 151.550000 2.00 151.550000 4.00 81.858300 11.00 120.000000 14.00 120.000000 15.00 211.337500 16.00 61.293067 17.00 92.261100 18.00 169.612500 19.00 92.692500 21.00 139.206933 22.00 91.656660 23.00 146.544433 24.00 122.997614 25.00 99.356967 26.00 54.425000 27.00 92.957300 28.00 47.830200 29.00 102.645833 30.00 67.017367 31.00 87.527500 32.00 53.395850 33.00 58.650000 34.00 26.550000 35.00 165.744911 36.00 125.623611 37.00 45.118067 38.00 103.711800 39.00 65.918320 40.00 69.336660 ... 3 31.00 11.216071 32.00 17.335758 33.00 10.844787 34.00 9.248950 34.50 6.437500 35.00 9.736800 36.00 12.081933 37.00 8.756250 38.00 13.748950 39.00 21.945833 40.00 13.599160 40.50 11.125000 41.00 20.283325 42.00 8.066675 43.00 20.466667 44.00 10.031250 45.00 13.025840 45.50 7.225000 47.00 10.250000 48.00 21.114600 49.00 0.000000 50.00 8.050000 51.00 7.618067 55.50 8.050000 59.00 7.250000 61.00 6.237500 63.00 9.587500 65.00 7.750000 70.50 7.750000 74.00 7.775000 Name: Fare, Length: 182, dtype: float64
# 写入代码 #result = pd.merge(means,survived_sex,on='Sex') result = means.append(survived_sex) result result.to_csv('sex_fare_survived.csv')
# 写入代码 survived_age = text['Survived'].groupby(text['Age']).sum() survived_age.head()
Age 0.42 1 0.67 1 0.75 2 0.83 2 0.92 1 Name: Survived, dtype: int64
找出存活人数的最高的年龄
# 写入代码 找出存活人数的最高的年龄 survived_age[survived_age.values==survived_age.max()]
Age 24.0 15 Name: Survived, dtype: int64
# 写入代码 计算存活人数最高的存活率(存活人数/总人数) rsum = text['Survived'].sum() precetn =survived_age.max()/rsum precetn
0.043859649122807015
text = pd.read_csv(‘result.csv’)
text.head()
<div> <style> .dataframe thead tr:only-child th { text-align: right; } .dataframe thead th { text-align: left; } .dataframe tbody tr th { vertical-align: top; } </style> <table border="1" class="dataframe"> <thead> <tr style="text-align: right;"> <th></th> <th>Unnamed: 0</th> <th>PassengerId</th> <th>Survived</th> <th>Pclass</th> <th>Name</th> <th>Sex</th> <th>Age</th> <th>SibSp</th> <th>Parch</th> <th>Ticket</th> <th>Fare</th> <th>Cabin</th> <th>Embarked</th> </tr> </thead> <tbody> <tr> <th>0</th> <td>0</td> <td>1</td> <td>0</td> <td>3</td> <td>Braund, Mr. Owen Harris</td> <td>male</td> <td>22.0</td> <td>1</td> <td>0</td> <td>A/5 21171</td> <td>7.2500</td> <td>NaN</td> <td>S</td> </tr> <tr> <th>1</th> <td>1</td> <td>2</td> <td>1</td> <td>1</td> <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td> <td>female</td> <td>38.0</td> <td>1</td> <td>0</td> <td>PC 17599</td> <td>71.2833</td> <td>C85</td> <td>C</td> </tr> <tr> <th>2</th> <td>2</td> <td>3</td> <td>1</td> <td>3</td> <td>Heikkinen, Miss. Laina</td> <td>female</td> <td>26.0</td> <td>0</td> <td>0</td> <td>STON/O2. 3101282</td> <td>7.9250</td> <td>NaN</td> <td>S</td> </tr> <tr> <th>3</th> <td>3</td> <td>4</td> <td>1</td> <td>1</td> <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td> <td>female</td> <td>35.0</td> <td>1</td> <td>0</td> <td>113803</td> <td>53.1000</td> <td>C123</td> <td>S</td> </tr> <tr> <th>4</th> <td>4</td> <td>5</td> <td>0</td> <td>3</td> <td>Allen, Mr. William Henry</td> <td>male</td> <td>35.0</td> <td>0</td> <td>0</td> <td>373450</td> <td>8.0500</td> <td>NaN</td> <td>S</td> </tr> </tbody> </table> </div> # 2 第二章:数据重构 ## 第一部分:数据聚合与运算 ### 2.6 数据运用 #### 2.6.1 任务一:通过教材《Python for Data Analysis》P303、Google or anything来学习了解GroupBy机制 #写入心得 groupby()用于分组计算,最基本的就是组内计数, 求和, 求均值, 求方差 #### 2.4.2:任务二:计算泰坦尼克号男性与女性的平均票价 ```python # 写入代码 mean_ave = text['Fare'].groupby(text['Sex']).mean() mean_ave
Sex female 44.479818 male 25.523893 Name: Fare, dtype: float64
在了解GroupBy机制之后,运用这个机制完成一系列的操作,来达到我们的目的。
下面通过几个任务来熟悉GroupBy机制。
# 写入代码 survived_sex = text['Survived'].groupby(text['Sex']).sum() survived_sex.head()
Sex female 233 male 109 Name: Survived, dtype: int64
# 写入代码 survived_pclass = text['Survived'].groupby(text['Pclass']).sum() survived_pclass
Pclass 1 136 2 87 3 119 Name: Survived, dtype: int64
【提示:】表中的存活那一栏,可以发现如果还活着记为1,死亡记为0
【思考】从数据分析的角度,上面的统计结果可以得出那些结论
#思考心得
存活人数中,一等舱较多,票价较高,且多为女性
【思考】从任务二到任务三中,这些运算可以通过agg()函数来同时计算。并且可以使用rename函数修改列名。你可以按照提示写出这个过程吗?
#思考心得 text.groupby('Sex').agg({'Fare': 'mean', 'Pclass': 'count'}).rename( columns= {'Fare': 'mean_fare', 'Pclass': 'count_pclass'})
mean_fare | count_pclass | |
---|---|---|
Sex | ||
female | 44.479818 | 314 |
male | 25.523893 | 577 |
# 写入代码 text.groupby(['Pclass','Age'])['Fare'].mean()
Pclass Age 1 0.92 151.550000 2.00 151.550000 4.00 81.858300 11.00 120.000000 14.00 120.000000 15.00 211.337500 16.00 61.293067 17.00 92.261100 18.00 169.612500 19.00 92.692500 21.00 139.206933 22.00 91.656660 23.00 146.544433 24.00 122.997614 25.00 99.356967 26.00 54.425000 27.00 92.957300 28.00 47.830200 29.00 102.645833 30.00 67.017367 31.00 87.527500 32.00 53.395850 33.00 58.650000 34.00 26.550000 35.00 165.744911 36.00 125.623611 37.00 45.118067 38.00 103.711800 39.00 65.918320 40.00 69.336660 ... 3 31.00 11.216071 32.00 17.335758 33.00 10.844787 34.00 9.248950 34.50 6.437500 35.00 9.736800 36.00 12.081933 37.00 8.756250 38.00 13.748950 39.00 21.945833 40.00 13.599160 40.50 11.125000 41.00 20.283325 42.00 8.066675 43.00 20.466667 44.00 10.031250 45.00 13.025840 45.50 7.225000 47.00 10.250000 48.00 21.114600 49.00 0.000000 50.00 8.050000 51.00 7.618067 55.50 8.050000 59.00 7.250000 61.00 6.237500 63.00 9.587500 65.00 7.750000 70.50 7.750000 74.00 7.775000 Name: Fare, Length: 182, dtype: float64
# 写入代码 #result = pd.merge(means,survived_sex,on='Sex') result = means.append(survived_sex) result result.to_csv('sex_fare_survived.csv')
# 写入代码 survived_age = text['Survived'].groupby(text['Age']).sum() survived_age.head()
Age 0.42 1 0.67 1 0.75 2 0.83 2 0.92 1 Name: Survived, dtype: int64
找出存活人数的最高的年龄
# 写入代码 找出存活人数的最高的年龄 survived_age[survived_age.values==survived_age.max()]
Age 24.0 15 Name: Survived, dtype: int64
# 写入代码 计算存活人数最高的存活率(存活人数/总人数) rsum = text['Survived'].sum() precetn =survived_age.max()/rsum precetn
0.043859649122807015