Exploratory Data Analysis with Titanic

dataset
In [1]: #Import the required libraries
import numpy as np

import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sb
%matplotlib inline

In [2]: #Load the data set

train_data=pd.read_csv('titanic_train_dataset.csv')

In [3]: #Display first 10 rows of dataset.

train_data.head(10)
Out[3]: PassengerId Survived Pclass Name Sex Age SibSp Parch Ticket Far

Braund,
0 1 0 3 Mr. Owen male 22.0 1 0 A/5 21171 7.250
Harris

Cumings,
Mrs. John
Bradley
1 2 1 1 female 38.0 1 0 PC 17599 71.283
(Florence
Briggs
Th...

Heikkinen,
STON/O2.
2 3 1 3 Miss. female 26.0 0 0 7.925
3101282
Laina

Futrelle,
Mrs.
Jacques
3 4 1 1 female 35.0 1 0 113803 53.100
Heath
(Lily May
Peel)

Allen, Mr.
4 5 0 3 William male 35.0 0 0 373450 8.050
Henry

Moran,
5 6 0 3 male NaN 0 0 330877 8.458
Mr. James

McCarthy,
6 7 0 1 Mr. male 54.0 0 0 17463 51.862
Timothy J

Palsson,
Master.
7 8 0 3 male 2.0 3 1 349909 21.075
Gosta
Leonard

Johnson,
Mrs.
Oscar W
8 9 1 3 female 27.0 0 2 347742 11.133
(Elisabeth
Vilhelmina
Berg)

Nasser,
Mrs.
9 10 1 2 Nicholas female 14.0 1 0 237736 30.070
(Adele
Achem)

In [6]: #Display last 5 rows of dataset.

train_data.tail()
Out[6]: PassengerId Survived Pclass Name Sex Age SibSp Parch Ticket Fare

Montvila,
886 887 0 2 Rev. male 27.0 0 0 211536 13.00
Juozas

Graham,
Miss.
887 888 1 1 female 19.0 0 0 112053 30.00
Margaret
Edith

Johnston,
Miss.
W./C.
888 889 0 3 Catherine female NaN 1 2 23.45
6607
Helen
"Carrie"

Behr, Mr.
889 890 1 1 Karl male 26.0 0 0 111369 30.00
Howell

Dooley,
890 891 0 3 Mr. male 32.0 0 0 370376 7.75
Patrick

In [5]: #Get the features of data set.

train_data.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 891 entries, 0 to 890
Data columns (total 12 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 PassengerId 891 non-null int64
1 Survived 891 non-null int64
2 Pclass 891 non-null int64
3 Name 891 non-null object
4 Sex 891 non-null object
5 Age 714 non-null float64
6 SibSp 891 non-null int64
7 Parch 891 non-null int64
8 Ticket 891 non-null object
9 Fare 891 non-null float64
10 Cabin 204 non-null object
11 Embarked 889 non-null object
dtypes: float64(2), int64(5), object(5)
memory usage: 83.7+ KB

Survived - Survival (0 = No; 1 = Yes) Pclass - Passenger Class (1 = 1st; 2 = 2nd; 3 =

3rd) Name - Name of the passenger Sex - Sex Age - Age

SibSp - Sibsp: The dataset defines family relations... Sibling = brother, sister,
stepbrother, stepsister Spouse = husband, wife

Parch: The dataset defines family relations in this way... Parent = mother, father Child =
daughter, son, stepdaughter, stepson Some children travelled only with a nanny,
therefore parch=0 for them.

Ticket - Ticket Number Fare - Passenger Fare Cabin - Cabin Embarked - Port of
Embarkation (C = Cherbourg; Q = Queenstown; S = Southampton)

In [6]: #4.Find the total number of rows and columns in the dataset.
 train_data.shape

(891, 12)
Out[6]:

In [21]: #Find the mean,std,count,min,max and percentiles of dataset.

train_data.describe()

Out[21]: PassengerId Survived Pclass Age SibSp Parch Fare

count 891.000000 891.000000 891.000000 891.000000 891.000000 891.000000 891.000000

mean 446.000000 0.383838 2.308642 29.856532 0.523008 0.381594 32.204208

std 257.353842 0.486592 0.836071 13.292134 1.102743 0.806057 49.693429

min 1.000000 0.000000 1.000000 0.420000 0.000000 0.000000 0.000000

25% 223.500000 0.000000 2.000000 22.000000 0.000000 0.000000 7.910400

50% 446.000000 0.000000 3.000000 30.000000 0.000000 0.000000 14.454200

75% 668.500000 1.000000 3.000000 36.000000 1.000000 0.000000 31.000000

max 891.000000 1.000000 3.000000 80.000000 8.000000 6.000000 512.329200

In [7]: #Check the null values in the data set.

train_data.isnull().sum()

PassengerId 0
Out[7]:
Survived 0
Pclass 0
Name 0
Sex 0
Age 177
SibSp 0
Parch 0
Ticket 0
Fare 0
Cabin 687
Embarked 2
dtype: int64

In [23]: #Draw the count plot to show the passengers survived or not survived.

#A countplot counts the categories and returns a count of their occurrenc

#As it only returns the count based of a categorical column,
#we need to specify only the x parameter.

sb.countplot('Survived',hue='Survived',data=train_data)
plt.show()

C:\ProgramData\Anaconda3\lib\site-packages\seaborn\_decorators.py:36: Fut
ureWarning: Pass the following variable as a keyword arg: x. From version
0.12, the only valid positional argument will be `data`, and passing othe
r arguments without an explicit keyword will result in an error or misint
erpretation.
warnings.warn(
In [26]: #Identify the number of male and female survived and not survived.

train_data.groupby(['Sex', 'Survived'])['Survived'].count()

Sex Survived
Out[26]:
female 0 81
1 233
male 0 468
1 109
Name: Survived, dtype: int64

In [12]: #Plot to show a passenger class has any impact on survived vs dead.
#train_data[['Sex','Survived']].groupby(['Sex']).mean().plot.bar()
sb.countplot('Sex',hue='Survived',data=train_data)
plt.show()

In [9]: sb.countplot('Pclass', hue='Survived', data=train_data)

plt.title('Pclass: Survived vs Dead')
plt.show()
 In [ ]: #it is clearly seen that pasangers of Class 1 are given high priority whi
#Rescue. There are greater number of passangers in Class 3 than Class 1 a
#Class 2 but very few, almost 25% in Class 3 survived. In Class 2, surviv
#and non-survival rate is 49% and 51% approx. While in Class 1 almost 68%
#people survived.

In [10]: #Identify the number of male and female survived or died based on the pas
#class.

# Compute a simple cross-tabulation of two (or more) factors. By default

#computes a frequency table of the factors unless an array of values and
#aggregation function are passed.

pd.crosstab([train_data.Sex,train_data.Survived],train_data.Pclass)

Out[10]: Pclass 1 2 3

Sex Survived

female 0 3 6 72

1 91 70 72

male 0 77 91 300

1 45 17 47

In [11]: #Find the age of oldest,youngest and average age of person travelled.
print('Age of oldest person travelled :',train_data['Age'].max())
print('Age of youngest person travelled :',train_data['Age'].min())
print('Average Age of person travelled :',train_data['Age'].mean())

Age of oldest person travelled : 80.0

Age of youngest person travelled : 0.42
Average Age of person travelled : 29.69911764705882

In [27]: train_data['Initial']=0
for i in train_data:
train_data['Initial']=train_data.Name.str.extract('([A-Za-z]+)\.') #e

In [28]: pd.crosstab(train_data.Initial,train_data.Sex)
Out[28]: Sex female male

Initial

Capt 0 1

Col 0 2

Countess 1 0

Don 0 1

Dr 1 6

Jonkheer 0 1

Lady 1 0

Major 0 2

Master 0 40

Miss 182 0

Mlle 2 0

Mme 1 0

Mr 0 517

Mrs 125 0

Ms 1 0

Rev 0 6

Sir 0 1

In [14]: train_data.groupby('Initial')['Age'].mean()

Initial
Out[14]:
Capt 70.000000
Col 58.000000
Countess 33.000000
Don 40.000000
Dr 42.000000
Jonkheer 38.000000
Lady 48.000000
Major 48.500000
Master 4.574167
Miss 21.773973
Mlle 24.000000
Mme 24.000000
Mr 32.368090
Mrs 35.898148
Ms 28.000000
Rev 43.166667
Sir 49.000000
Name: Age, dtype: float64

In [15]: train_data['Initial'].replace(['Capt','Col','Countess','Don','Dr','Jonkhe
'Mr','Miss','Mr','Other','Mr','Mrs','Mr','

In [ ]:

In [16]: train_data['Initial'].replace(['Mlle','Mme','Ms','Dr','Major','Lady','Cou
'Miss','Miss','Mr','Mr','Mrs','Mrs','Othe
In [17]: #Average age based on initials
train_data.groupby('Initial')['Age'].mean()

Initial
Out[17]:
Master 4.574167
Miss 21.879195
Mr 32.891990
Mrs 35.828829
Other 42.000000
Name: Age, dtype: float64

In [19]: #Fill the null values of age with average age based on initial
train_data.loc[(train_data.Age.isnull()) & (train_data.Initial=='Mr'),'Ag
train_data.loc[(train_data.Age.isnull()) & (train_data.Initial=='Mrs'),'A
train_data.loc[(train_data.Age.isnull()) & (train_data.Initial=='Master')
train_data.loc[(train_data.Age.isnull()) & (train_data.Initial=='Miss'),'
train_data.loc[(train_data.Age.isnull()) & (train_data.Initial=='Other'),

In [27]: train_data.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 891 entries, 0 to 890
Data columns (total 13 columns):
PassengerId 891 non-null int64
Survived 891 non-null int64
Pclass 891 non-null int64
Name 891 non-null object
Sex 891 non-null object
Age 891 non-null float64
SibSp 891 non-null int64
Parch 891 non-null int64
Ticket 891 non-null object
Fare 891 non-null float64
Cabin 204 non-null object
Embarked 889 non-null object
Initial 891 non-null object
dtypes: float64(2), int64(5), object(6)
memory usage: 90.6+ KB

In [28]: train_data.Age.isnull().any()

False
Out[28]:

In [20]: f,ax=plt.subplots(1,2,figsize=(20,20))
train_data[train_data['Survived']==0].Age.plot.hist(ax=ax[0],bins=20,edge
ax[0].set_title('Survived = 0')
x1=list(range(0,85,5))
ax[0].set_xticks(x1)
train_data[train_data['Survived']==1].Age.plot.hist(ax=ax[1],bins=20,edge
x2=list(range(0,85,5))
ax[1].set_xticks(x2)
ax[1].set_title('Survived = 1')
plt.show()
 In [ ]: #Observations: (1) First priority during Rescue is given to children and
#as the persons<5 are save by large numbers (2) The oldest saved passange
# of age 80 (3) The most deaths were between 30-40

In [31]: #Identify the number of passenger died based on size of the family
#(using SibSp feature) and also draw the plot..
pd.crosstab([train_data.SibSp],train_data.Survived)

Out[31]: Survived 0 1

SibSp

0 398 210

1 97 112

2 15 13

3 12 4

4 15 3

5 5 0

8 7 0

In [29]: #Identify the number of passenger died based on size of the family
#(using SibSp feature) and also draw the plot..
pd.crosstab([train_data.SibSp],train_data.Survived).style.background_grad
Out[29]: Survived 0 1

SibSp

0 398 210

1 97 112

2 15 13

3 12 4

4 15 3

5 5 0

8 7 0

In [41]: sb.barplot('SibSp','Survived', data=train_data)

<matplotlib.axes._subplots.AxesSubplot at 0x24c61b50748>
Out[41]:

In [ ]: #The graph shows that if a passanger is alone in ship with no siblings, s

#The graph decreases as no of siblings increase. This is interesting beca
#I will save them instead of saving myself.
#But there's something wrong, the survival rate for families with 5-8 mem
#Is this because of PClass? Yes this is PClass, The crosstab shows that P
#were all in Pclass3. It is imminent that all the large families in Pclas

In [39]: f,ax=plt.subplots(1,2,figsize=(20,8))
sb.barplot('SibSp','Survived', data=train_data,ax=ax[0])
ax[0].set_title('SipSp vs Survived in BarPlot')
plt.show()
In [40]: f,ax=plt.subplots(1,2,figsize=(20,8))
sb.barplot('SibSp','Survived', data=train_data,ax=ax[0])
ax[0].set_title('SipSp vs Survived in BarPlot')
sb.factorplot('SibSp','Survived', data=train_data,ax=ax[1])
ax[1].set_title('SibSp vs Survived in FactorPlot')
plt.close(2)
plt.show()

In [33]: pd.crosstab(train_data.SibSp,train_data.Pclass).style.background_gradient

Out[33]: Pclass 1 2 3

SibSp

0 137 120 351

1 71 55 83

2 5 8 15

3 3 1 12

4 0 0 18

5 0 0 5

8 0 0 7

In [ ]: #Barplot and Crosstab data shows that if a passanger is alone in ship wit
#siblings, survival rate is 34.5%. The graph decreases as no of siblings
#increase. This is interesting because, If I have a family onboard, I wil
#them instead of saving myself. But there's something wrong, the survival
#for families with 5-8 members is 0%. Is this because of PClass?
 #Yes this is PClass, The crosstab shows that Person with SibSp>3 were all
#Pclass3. It is imminent that all the large families in Pclass3(>3) died.

In [64]: train_data.corr(method='pearson')

Out[64]: PassengerId Survived Pclass Age SibSp Parch Fare

PassengerId 1.000000 -0.005007 -0.035144 0.041709 -0.057527 -0.001652 0.012658

Survived -0.005007 1.000000 -0.338481 -0.091497 -0.035322 0.081629 0.257307

Pclass -0.035144 -0.338481 1.000000 -0.339582 0.083081 0.018443 -0.549500

Age 0.041709 -0.091497 -0.339582 1.000000 -0.267734 -0.198712 0.089087

SibSp -0.057527 -0.035322 0.083081 -0.267734 1.000000 0.414838 0.159651

Parch -0.001652 0.081629 0.018443 -0.198712 0.414838 1.000000 0.216225

Fare 0.012658 0.257307 -0.549500 0.089087 0.159651 0.216225 1.000000

In [ ]: #From above correlation table we can see that Survival is inversly correl
#Pclass value. In this case since Class 1 has lower numerical value, it h
#better survival rate compared to other classes.

#We also see that Age and Survival are slighltly correlated.

In [70]: train_data.groupby(['Survived']).hist()

sb.factorplot('Survived', data=train_data, kind='count')

<seaborn.axisgrid.FacetGrid at 0x27a34bf3c50>
Out[70]:
In [78]: #Plot Agewise distribution of the passenger aboard.
sb.distplot(train_data['Age'].dropna(), bins=15,kde=False)

<matplotlib.axes._subplots.AxesSubplot at 0x27a35629b70>
Out[78]:

In [ ]: #Note: Many passensgers are of age 15-40 yrs. But this is not complete da

In [83]: #Age wise Distribution of Male and Female passengers

plt.hist(train_data['Age'][(train_data['Sex'] == 'female')].dropna(), bin
plt.hist(train_data['Age'][(train_data['Sex'] == 'male')].dropna(), bins=
plt.xlabel('Age')
 plt.ylabel('Count')
plt.title('Age wise Distribution of Male and Female passengers')
plt.show()

In [ ]: #There were many male passengers aboared compared to female passengers.

In [85]: #survival based on passenger’s class for both genders.

grouped_by_pclass = train_data.groupby(['Pclass', 'Survived', 'Sex'])
grouped_by_pclass.size()

Pclass Survived Sex

Out[85]:
1 0 female 3
male 77
1 female 91
male 45
2 0 female 6
male 91
1 female 70
male 17
3 0 female 72
male 300
1 female 72
male 47
dtype: int64

In [ ]: