Created by SmirkyGraphs. Code: GitHub. Source: Kaggle.
Introduction¶
Index¶
1) Introduction
- Index
- Import Libraries
- Load data
- Variable Details/Summary
2) Correlation
- Correlation of All Values
- Correlation with Target Variable
- Values
- Categories
- Figure out missing value columns
- Fill out missing values
5) Predicting
- Encode categories
- Split train/test
- Predicting prices
- Exporting predictions
Importing Libraries¶
In [1]:
# For Data
import pandas as pd
from pandas import Series, DataFrame
import numpy as np
# For Visualizations
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
my_color = sns.color_palette()
sns.set_style('darkgrid')
sns.set()
Loading the Data¶
In [2]:
df = pd.read_csv('train.csv')
Variable Details¶
In [3]:
df.head()
Out[3]:
In [4]:
df.shape
Out[4]:
We have 1460 samples made of 80 features and 1 target variable that is the SalePrice to work with
In [5]:
test = pd.read_csv('test.csv')
In [6]:
test.head()
Out[6]:
In [7]:
test.shape
Out[7]:
In [8]:
df.info()
In [9]:
df.head()
Out[9]:
In [10]:
# Statistical Summary of Target
df['SalePrice'].describe()
Out[10]:
In [11]:
# Statistical Summary
df.describe()
Out[11]:
Correlation¶
Correlation of All Variables¶
In [12]:
# Compute the correlation matrix
corr = df.corr()
# Set up the matplotlib figure
f, ax = plt.subplots(figsize=(11, 9))
# Custom colormap
cmap = sns.diverging_palette(220, 10, as_cmap=True)
# Draw the heatmap
sns.heatmap(corr, cmap=cmap, vmax=1, square=True, linewidths=.5)
Out[12]:
Correlation With Target Variable¶
In [13]:
corr = df.corr()["SalePrice"]
corr[np.argsort(corr, axis=0)[::-1]]
# Top 35 Features
corr.sort_values(ascending=False).head(35)
Out[13]:
Visualisations¶
Values Visualizations¶
In [14]:
df['SalePrice'].hist(bins=70)
Out[14]:
In [15]:
df['YearBuilt'].hist(bins=70)
Out[15]:
In [16]:
sns.regplot(x = 'OverallQual', y = 'SalePrice', data = df, color = 'g')
Out[16]:
In [17]:
sns.regplot(x = 'OverallCond', y = 'SalePrice', data = df, color = 'g')
Out[17]:
In [18]:
sns.regplot(x = 'YearBuilt', y = 'SalePrice', data = df, color = 'g')
Out[18]:
In [19]:
# Getting Price per Sqrft
df['PricePrSF'] = df['SalePrice']/df['GrLivArea']
In [20]:
df['PricePrSF'].hist(bins=70)
Out[20]:
In [21]:
df['PricePrSF'].describe()
Out[21]:
In [22]:
x, axarr = plt.subplots(3, 2, figsize=(10, 9))
price = df.SalePrice.values
axarr[0, 0].scatter(df.GrLivArea.values, price)
axarr[0, 0].set_title('GrLiveArea')
axarr[0, 1].scatter(df.GarageArea.values, price)
axarr[0, 1].set_title('GarageArea')
axarr[1, 0].scatter(df.TotalBsmtSF.values, price)
axarr[1, 0].set_title('TotalBsmtSF')
axarr[1, 1].scatter(df['1stFlrSF'].values, price)
axarr[1, 1].set_title('1stFlrSF')
axarr[2, 0].scatter(df.TotRmsAbvGrd.values, price)
axarr[2, 0].set_title('TotRmsAbvGrd')
axarr[2, 1].scatter(df.MasVnrArea.values, price)
axarr[2, 1].set_title('MasVnrArea')
x.text(-0.01, 0.5, 'Sale Price', va='center', rotation='vertical', fontsize = 12)
plt.tight_layout()
plt.show()
Categories Visualizations¶
In [23]:
sns.factorplot('SaleCondition',data=df,kind="count")
Out[23]:
In [24]:
sns.factorplot('SaleType',data=df,kind="count")
Out[24]:
In [25]:
sns.factorplot('Street',data=df,kind="count")
Out[25]:
In [26]:
sns.factorplot('Alley',data=df,kind="count")
Out[26]:
In [27]:
sns.factorplot('BldgType',data=df,kind="count")
Out[27]:
In [28]:
sns.factorplot('HouseStyle',data=df,kind="count")
Out[28]:
In [29]:
sns.factorplot('OverallQual',data=df,kind="count")
Out[29]:
In [30]:
plt.figure(figsize = (12, 6))
sns.countplot(x = 'Neighborhood', data = df)
xt = plt.xticks(rotation=45)
In [31]:
fig, ax = plt.subplots(2, 2, figsize = (10, 8))
sns.boxplot('BsmtCond', 'SalePrice', data = df, ax = ax[0, 0])
sns.boxplot('BsmtQual', 'SalePrice', data = df, ax = ax[0, 1])
sns.boxplot('BsmtExposure', 'SalePrice', data = df, ax = ax[1, 0])
sns.boxplot('BsmtFinType1', 'SalePrice', data = df, ax = ax[1, 1])
Out[31]:
In [32]:
fig, ax = plt.subplots(1, 2, figsize = (12,6))
sns.boxplot('HouseStyle', 'SalePrice', data = df, ax = ax[0])
sns.boxplot('BldgType', 'SalePrice', data = df, ax = ax[1])
Out[32]:
In [33]:
sns.factorplot('HeatingQC',data=df,kind="count")
Out[33]:
In [34]:
sns.factorplot('Fence',data=df,kind="count")
Out[34]:
In [35]:
sns.factorplot('HeatingQC', 'SalePrice', hue = 'CentralAir', estimator = np.mean, data = df,
size = 4.5, aspect = 1.4)
Out[35]:
In [36]:
sns.factorplot('BedroomAbvGr',data=df,kind="count")
Out[36]:
Missing Values¶
In [37]:
train = df
# Joining train and test
features = pd.concat([train, test], keys=['train', 'test'])
Finding Missing Values¶
In [38]:
features.isnull().sum().sort_values(ascending=False).head(35)
Out[38]:
Replacing Missing Values¶
In [39]:
# Removing Outliers
features.drop(features[(features['GrLivArea']>4000) & (features['SalePrice']<300000)].index)
# For these missing features missing values likely mean it doesn't exist
features['PoolQC'] = features['PoolQC'].fillna('None')
features['MiscFeature'] = features['MiscFeature'].fillna('None')
features['Alley'] = features['Alley'].fillna('None')
features['Fence'] = features['Fence'].fillna('None')
features['FireplaceQu'] = features['FireplaceQu'].fillna('None')
features['GarageType'] = features['GarageType'].fillna('None')
features['GarageFinish'] = features['GarageFinish'].fillna('None')
features['GarageQual'] = features['GarageQual'].fillna('None')
features['GarageCond'] = features['GarageCond'].fillna('None')
features['BsmtQual'] = features['BsmtQual'].fillna('None')
features['BsmtCond'] = features['BsmtCond'].fillna('None')
features['BsmtExposure'] = features['BsmtExposure'].fillna('None')
features['BsmtFinType1'] = features['BsmtFinType1'].fillna('None')
features['BsmtFinType2'] = features['BsmtFinType2'].fillna('None')
features['MasVnrType'] = features['MasVnrType'].fillna('None')
features['MSSubClass'] = features['MSSubClass'].fillna('None')
# Fill missing with 0
features['TotalBsmtSF'] = features['TotalBsmtSF'].fillna(0)
features['GarageYrBlt'] = features['GarageYrBlt'].fillna(0)
features['GarageArea'] = features['GarageArea'].fillna(0.0)
features['GarageCars'] = features['GarageCars'].fillna(0.0)
features['MasVnrArea'] = features['MasVnrArea'].fillna(0.0)
features['BsmtFullBath'] = features['BsmtFullBath'].fillna(0)
features['BsmtHalfBath'] = features['BsmtHalfBath'].fillna(0)
features['BsmtUnfSF'] = features['BsmtUnfSF'].fillna(0)
features['BsmtFinSF1'] = features['BsmtFinSF1'].fillna(0)
features['BsmtFinSF2'] = features['BsmtFinSF2'].fillna(0)
# Replace with mode
features['MSZoning'] = features['MSZoning'].fillna(features['MSZoning'].mode()[0])
features['Electrical'] = features['Electrical'].fillna(features['Electrical'].mode()[0])
features['KitchenQual'] = features['KitchenQual'].fillna(features['KitchenQual'].mode()[0])
features['SaleType'] = features['SaleType'].fillna(features['SaleType'].mode()[0])
features['Functional'] = features['Functional'].fillna(features['Functional'].mode()[0])
features['Exterior1st'] = features['Exterior1st'].fillna(features['Exterior1st'].mode()[0])
features['Exterior2nd'] = features['Exterior2nd'].fillna(features['Exterior2nd'].mode()[0])
features['Utilities'] = features['Utilities'].fillna(features['Utilities'].mode()[0])
# Replace with median
features['LotFrontage'] = features['LotFrontage'].fillna(features['LotFrontage'].median())
# Adding total sqfootage
features['TotalSF'] = features['TotalBsmtSF'] + features['1stFlrSF'] + features['2ndFlrSF']
# Dropping SF
features = features.drop(['TotalBsmtSF'], axis=1)
features = features.drop(['1stFlrSF'], axis=1)
features = features.drop(['2ndFlrSF'], axis=1)
In [40]:
features.dtypes.value_counts()
Out[40]:
In [41]:
features.isnull().sum().sort_values(ascending=False).head(3)
Out[41]:
In [42]:
features.shape
Out[42]:
Predicting Sales Prices¶
Replacing Categorical¶
In [44]:
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import cross_val_score, train_test_split, KFold
from sklearn.metrics import mean_squared_log_error, mean_squared_error, make_scorer, mean_absolute_error
# Models
from sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor, AdaBoostRegressor
from sklearn.linear_model import ElasticNetCV
from sklearn.tree import ExtraTreeRegressor, DecisionTreeRegressor
from sklearn.svm import SVR
from civismlext.stacking import StackedRegressor
# Stacking
from civismlext.nonnegative import NonNegativeLinearRegression
# Xgboost
import xgboost as xgb
Encoding Categories¶
In [45]:
le = LabelEncoder()
features['BldgType'] = le.fit_transform(features['BldgType'])
features['BsmtCond'] = le.fit_transform(features['BsmtCond'])
features['BsmtExposure'] = le.fit_transform(features['BsmtExposure'])
features['BsmtFinType1'] = le.fit_transform(features['BsmtFinType1'])
features['BsmtFinType2'] = le.fit_transform(features['BsmtFinType2'])
features['BsmtQual'] = le.fit_transform(features['BsmtQual'])
features['CentralAir'] = le.fit_transform(features['CentralAir'])
features['Condition1'] = le.fit_transform(features['Condition1'])
features['Condition2'] = le.fit_transform(features['Condition2'])
features['Electrical'] = le.fit_transform(features['Electrical'])
features['ExterCond'] = le.fit_transform(features['ExterCond'])
features['ExterQual'] = le.fit_transform(features['ExterQual'])
features['Exterior1st'] = le.fit_transform(features['Exterior1st'])
features['Exterior2nd'] = le.fit_transform(features['Exterior2nd'])
features['FireplaceQu'] = le.fit_transform(features['FireplaceQu'])
features['Foundation'] = le.fit_transform(features['Foundation'])
features['Functional'] = le.fit_transform(features['Functional'])
features['GarageCond'] = le.fit_transform(features['GarageCond'])
features['GarageFinish'] = le.fit_transform(features['GarageFinish'])
features['GarageQual'] = le.fit_transform(features['GarageQual'])
features['GarageType'] = le.fit_transform(features['GarageType'])
features['Heating'] = le.fit_transform(features['Heating'])
features['HeatingQC'] = le.fit_transform(features['HeatingQC'])
features['HouseStyle'] = le.fit_transform(features['HouseStyle'])
features['KitchenQual'] = le.fit_transform(features['KitchenQual'])
features['LandContour'] = le.fit_transform(features['LandContour'])
features['LandSlope'] = le.fit_transform(features['LandSlope'])
features['LotConfig'] = le.fit_transform(features['LotConfig'])
features['LotShape'] = le.fit_transform(features['LotShape'])
features['MSZoning'] = le.fit_transform(features['MSZoning'])
features['MasVnrType'] = le.fit_transform(features['MasVnrType'])
features['Neighborhood'] = le.fit_transform(features['Neighborhood'])
features['PavedDrive'] = le.fit_transform(features['PavedDrive'])
features['RoofMatl'] = le.fit_transform(features['RoofMatl'])
features['RoofStyle'] = le.fit_transform(features['RoofStyle'])
features['SaleCondition'] = le.fit_transform(features['SaleCondition'])
features['SaleType'] = le.fit_transform(features['SaleType'])
features['Street'] = le.fit_transform(features['Street'])
features['Utilities'] = le.fit_transform(features['Utilities'])
features['Fence'] = le.fit_transform(features['Fence'])
features['Alley'] = le.fit_transform(features['Alley'])
features['MiscFeature'] = le.fit_transform(features['MiscFeature'])
features['PoolQC'] = le.fit_transform(features['PoolQC'])
features['MSSubClass'] = le.fit_transform(features['MSSubClass'])
Splitting the Data¶
In [46]:
train = features.loc['train']
test = features.loc['test']
test = test.drop(['SalePrice'], axis=1)
test = test.drop(['PricePrSF'], axis=1)
train = train.drop(['PricePrSF'], axis=1)
In [47]:
print(train.shape)
print(test.shape)
In [48]:
# Splitting the data
X = train.drop(['SalePrice'], axis=1)
y = train.SalePrice
In [49]:
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=4,test_size=0.2)
XGBoost¶
In [50]:
xgb_model = xgb.XGBRegressor(n_estimators=400, n_jobs=-1, nthread=-1)
In [51]:
xgb_model.fit(X_train, y_train)
Out[51]:
In [52]:
xgb_model.score(X_train,y_train)
Out[52]:
In [53]:
xgb_model.score(X_test,y_test)
Out[53]:
Random Forest¶
In [54]:
rf = RandomForestRegressor(random_state=4, n_estimators=4000, n_jobs=-1)
rf.fit(X_train, y_train.ravel())
Out[54]:
In [55]:
rf.score(X_train,y_train)
Out[55]:
In [56]:
rf.score(X_test,y_test)
Out[56]:
Adabooster¶
In [57]:
ada = AdaBoostRegressor(n_estimators=400, learning_rate=0.1)
In [58]:
ada.fit(X_train, y_train.ravel())
Out[58]:
In [59]:
ada.score(X_train,y_train)
Out[59]:
In [60]:
ada.score(X_test,y_test)
Out[60]:
Non Negative Linear Regression¶
In [61]:
nn = NonNegativeLinearRegression(fit_intercept=True, normalize=False)
In [62]:
nn.fit(X_train, y_train)
Out[62]:
In [63]:
nn.score(X_train,y_train)
Out[63]:
In [64]:
nn.score(X_test,y_test)
Out[64]:
Extra Tree¶
In [65]:
ex = ExtraTreesRegressor(random_state=4, n_jobs=-1, max_features=50)
In [66]:
ex.fit(X_train, y_train)
Out[66]:
In [67]:
ex.score(X_train,y_train)
Out[67]:
In [68]:
ex.score(X_test,y_test)
Out[68]:
ElasticNetCV¶
In [69]:
en = ElasticNetCV(random_state=4, n_jobs=-1, alphas=[0.1, 1, 10, 100], cv=5)
In [70]:
en.fit(X_train, y_train)
Out[70]:
In [71]:
en.score(X_train,y_train)
Out[71]:
In [72]:
en.score(X_test,y_test)
Out[72]:
Combined Model¶
In [73]:
models = [
('ada', ada),
('en', en),
('nn', nn),
('ex', ex),
('rf', rf),
]
In [74]:
stack = StackedRegressor(models, n_jobs=-1)
In [75]:
stack.fit(X_train, y_train)
Out[75]:
In [76]:
stack.score(X_train,y_train)
Out[76]:
In [77]:
stack.score(X_test,y_test)
Out[77]:
In [78]:
XGB_model = xgb_model.fit(X_train, y_train)
In [79]:
y_pred = stack.predict(X_test)
print('MAE:\t$%.2f' % mean_absolute_error(y_test, y_pred))
print('MSLE:\t%.5f' % mean_squared_log_error(y_test, y_pred))
In [80]:
predictions = XGB_model.predict(test)*0.30 + stack.predict(test)*0.70
In [81]:
submission = pd.DataFrame({'Id': test.Id, 'SalePrice': predictions})
In [82]:
submission.to_csv('submissions.csv', index=False)