Import important libraries and packages

In [387]:
#Export important libraries
import warnings
warnings.filterwarnings(action="ignore")

#Data preprocessing and ploting
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
#Statistics
from scipy.stats import skew, norm
from scipy.stats import boxcox_normmax
from scipy.special import boxcox1p

#Misc
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import cross_val_score,LeaveOneOut,KFold
from sklearn.model_selection import RandomizedSearchCV,GridSearchCV
from sklearn.model_selection import train_test_split,cross_val_score
from sklearn.metrics import mean_squared_error,mean_absolute_error
from sklearn.preprocessing import OneHotEncoder,LabelEncoder,scale,StandardScaler,RobustScaler
from sklearn.pipeline import make_pipeline
from sklearn.decomposition import PCA

#Machine Learning Model
from sklearn.linear_model import LinearRegression,Lasso,Ridge, LassoCV, RidgeCV, ElasticNet, LassoLarsCV,ElasticNet
from sklearn.ensemble import RandomForestRegressor,GradientBoostingRegressor,AdaBoostRegressor,BaggingRegressor
from sklearn.kernel_ridge import KernelRidge
from lightgbm import LGBMRegressor
import lightgbm as lgm
from sklearn.svm import SVR
from sklearn.tree import DecisionTreeRegressor
from xgboost import XGBRFRegressor,XGBRegressor
from mlxtend.regressor import StackingCVRegressor

Acquire the training and testing data

In [388]:
train_df=pd.read_csv("train.csv")
test_df=pd.read_csv("test.csv")

Data Exploration Part

In [389]:
print("Shaping of training data",train_df.shape)
print("Shaping of testing data",test_df.shape)
Shaping of training data (1460, 81)
Shaping of testing data (1459, 80)
In [390]:
#See top 5 records of training data
train_df.head()
Out[390]:
Id MSSubClass MSZoning LotFrontage LotArea Street Alley LotShape LandContour Utilities ... PoolArea PoolQC Fence MiscFeature MiscVal MoSold YrSold SaleType SaleCondition SalePrice
0 1 60 RL 65.0 8450 Pave NaN Reg Lvl AllPub ... 0 NaN NaN NaN 0 2 2008 WD Normal 208500
1 2 20 RL 80.0 9600 Pave NaN Reg Lvl AllPub ... 0 NaN NaN NaN 0 5 2007 WD Normal 181500
2 3 60 RL 68.0 11250 Pave NaN IR1 Lvl AllPub ... 0 NaN NaN NaN 0 9 2008 WD Normal 223500
3 4 70 RL 60.0 9550 Pave NaN IR1 Lvl AllPub ... 0 NaN NaN NaN 0 2 2006 WD Abnorml 140000
4 5 60 RL 84.0 14260 Pave NaN IR1 Lvl AllPub ... 0 NaN NaN NaN 0 12 2008 WD Normal 250000

5 rows × 81 columns

In [391]:
#see last five training records
train_df.tail()
Out[391]:
Id MSSubClass MSZoning LotFrontage LotArea Street Alley LotShape LandContour Utilities ... PoolArea PoolQC Fence MiscFeature MiscVal MoSold YrSold SaleType SaleCondition SalePrice
1455 1456 60 RL 62.0 7917 Pave NaN Reg Lvl AllPub ... 0 NaN NaN NaN 0 8 2007 WD Normal 175000
1456 1457 20 RL 85.0 13175 Pave NaN Reg Lvl AllPub ... 0 NaN MnPrv NaN 0 2 2010 WD Normal 210000
1457 1458 70 RL 66.0 9042 Pave NaN Reg Lvl AllPub ... 0 NaN GdPrv Shed 2500 5 2010 WD Normal 266500
1458 1459 20 RL 68.0 9717 Pave NaN Reg Lvl AllPub ... 0 NaN NaN NaN 0 4 2010 WD Normal 142125
1459 1460 20 RL 75.0 9937 Pave NaN Reg Lvl AllPub ... 0 NaN NaN NaN 0 6 2008 WD Normal 147500

5 rows × 81 columns

In [392]:
#see top 5 records of test data
test_df.head()
Out[392]:
Id MSSubClass MSZoning LotFrontage LotArea Street Alley LotShape LandContour Utilities ... ScreenPorch PoolArea PoolQC Fence MiscFeature MiscVal MoSold YrSold SaleType SaleCondition
0 1461 20 RH 80.0 11622 Pave NaN Reg Lvl AllPub ... 120 0 NaN MnPrv NaN 0 6 2010 WD Normal
1 1462 20 RL 81.0 14267 Pave NaN IR1 Lvl AllPub ... 0 0 NaN NaN Gar2 12500 6 2010 WD Normal
2 1463 60 RL 74.0 13830 Pave NaN IR1 Lvl AllPub ... 0 0 NaN MnPrv NaN 0 3 2010 WD Normal
3 1464 60 RL 78.0 9978 Pave NaN IR1 Lvl AllPub ... 0 0 NaN NaN NaN 0 6 2010 WD Normal
4 1465 120 RL 43.0 5005 Pave NaN IR1 HLS AllPub ... 144 0 NaN NaN NaN 0 1 2010 WD Normal

5 rows × 80 columns

In [393]:
#see last 5 record of test data
test_df.tail()
Out[393]:
Id MSSubClass MSZoning LotFrontage LotArea Street Alley LotShape LandContour Utilities ... ScreenPorch PoolArea PoolQC Fence MiscFeature MiscVal MoSold YrSold SaleType SaleCondition
1454 2915 160 RM 21.0 1936 Pave NaN Reg Lvl AllPub ... 0 0 NaN NaN NaN 0 6 2006 WD Normal
1455 2916 160 RM 21.0 1894 Pave NaN Reg Lvl AllPub ... 0 0 NaN NaN NaN 0 4 2006 WD Abnorml
1456 2917 20 RL 160.0 20000 Pave NaN Reg Lvl AllPub ... 0 0 NaN NaN NaN 0 9 2006 WD Abnorml
1457 2918 85 RL 62.0 10441 Pave NaN Reg Lvl AllPub ... 0 0 NaN MnPrv Shed 700 7 2006 WD Normal
1458 2919 60 RL 74.0 9627 Pave NaN Reg Lvl AllPub ... 0 0 NaN NaN NaN 0 11 2006 WD Normal

5 rows × 80 columns

Data Preprocessing and Feature engineering

1. finding missing values

In [394]:
train_miss=train_df.isnull().sum()
test_miss=test_df.isnull().sum()
In [395]:
train_miss[train_miss>0]
Out[395]:
LotFrontage      259
Alley           1369
MasVnrType         8
MasVnrArea         8
BsmtQual          37
BsmtCond          37
BsmtExposure      38
BsmtFinType1      37
BsmtFinType2      38
Electrical         1
FireplaceQu      690
GarageType        81
GarageYrBlt       81
GarageFinish      81
GarageQual        81
GarageCond        81
PoolQC          1453
Fence           1179
MiscFeature     1406
dtype: int64
In [396]:
#visualize the missing value in training datasets
plt.subplots(figsize=(12, 9))
sns.set_style("whitegrid")
train_miss=train_miss[train_miss>0]
train_miss.sort_values(inplace=True)
train_miss.plot(kind='bar')
Out[396]:
<matplotlib.axes._subplots.AxesSubplot at 0x1c9898bba58>
In [397]:
test_miss=test_miss[test_miss>0]
test_miss
Out[397]:
MSZoning           4
LotFrontage      227
Alley           1352
Utilities          2
Exterior1st        1
Exterior2nd        1
MasVnrType        16
MasVnrArea        15
BsmtQual          44
BsmtCond          45
BsmtExposure      44
BsmtFinType1      42
BsmtFinSF1         1
BsmtFinType2      42
BsmtFinSF2         1
BsmtUnfSF          1
TotalBsmtSF        1
BsmtFullBath       2
BsmtHalfBath       2
KitchenQual        1
Functional         2
FireplaceQu      730
GarageType        76
GarageYrBlt       78
GarageFinish      78
GarageCars         1
GarageArea         1
GarageQual        78
GarageCond        78
PoolQC          1456
Fence           1169
MiscFeature     1408
SaleType           1
dtype: int64
In [398]:
#visualize missing value in test datasets
plt.subplots(figsize=(12, 9))
test_miss=test_miss[test_miss>0]
test_miss.sort_values(inplace=True)
test_miss.plot(kind='bar')
Out[398]:
<matplotlib.axes._subplots.AxesSubplot at 0x1c98994bb38>

Description of data

In [399]:
train_df.describe()
Out[399]:
Id MSSubClass LotFrontage LotArea OverallQual OverallCond YearBuilt YearRemodAdd MasVnrArea BsmtFinSF1 ... WoodDeckSF OpenPorchSF EnclosedPorch 3SsnPorch ScreenPorch PoolArea MiscVal MoSold YrSold SalePrice
count 1460.000000 1460.000000 1201.000000 1460.000000 1460.000000 1460.000000 1460.000000 1460.000000 1452.000000 1460.000000 ... 1460.000000 1460.000000 1460.000000 1460.000000 1460.000000 1460.000000 1460.000000 1460.000000 1460.000000 1460.000000
mean 730.500000 56.897260 70.049958 10516.828082 6.099315 5.575342 1971.267808 1984.865753 103.685262 443.639726 ... 94.244521 46.660274 21.954110 3.409589 15.060959 2.758904 43.489041 6.321918 2007.815753 180921.195890
std 421.610009 42.300571 24.284752 9981.264932 1.382997 1.112799 30.202904 20.645407 181.066207 456.098091 ... 125.338794 66.256028 61.119149 29.317331 55.757415 40.177307 496.123024 2.703626 1.328095 79442.502883
min 1.000000 20.000000 21.000000 1300.000000 1.000000 1.000000 1872.000000 1950.000000 0.000000 0.000000 ... 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 1.000000 2006.000000 34900.000000
25% 365.750000 20.000000 59.000000 7553.500000 5.000000 5.000000 1954.000000 1967.000000 0.000000 0.000000 ... 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 5.000000 2007.000000 129975.000000
50% 730.500000 50.000000 69.000000 9478.500000 6.000000 5.000000 1973.000000 1994.000000 0.000000 383.500000 ... 0.000000 25.000000 0.000000 0.000000 0.000000 0.000000 0.000000 6.000000 2008.000000 163000.000000
75% 1095.250000 70.000000 80.000000 11601.500000 7.000000 6.000000 2000.000000 2004.000000 166.000000 712.250000 ... 168.000000 68.000000 0.000000 0.000000 0.000000 0.000000 0.000000 8.000000 2009.000000 214000.000000
max 1460.000000 190.000000 313.000000 215245.000000 10.000000 9.000000 2010.000000 2010.000000 1600.000000 5644.000000 ... 857.000000 547.000000 552.000000 508.000000 480.000000 738.000000 15500.000000 12.000000 2010.000000 755000.000000

8 rows × 38 columns

In [400]:
test_df.describe()
Out[400]:
Id MSSubClass LotFrontage LotArea OverallQual OverallCond YearBuilt YearRemodAdd MasVnrArea BsmtFinSF1 ... GarageArea WoodDeckSF OpenPorchSF EnclosedPorch 3SsnPorch ScreenPorch PoolArea MiscVal MoSold YrSold
count 1459.000000 1459.000000 1232.000000 1459.000000 1459.000000 1459.000000 1459.000000 1459.000000 1444.000000 1458.000000 ... 1458.000000 1459.000000 1459.000000 1459.000000 1459.000000 1459.000000 1459.000000 1459.000000 1459.000000 1459.000000
mean 2190.000000 57.378341 68.580357 9819.161069 6.078821 5.553804 1971.357779 1983.662783 100.709141 439.203704 ... 472.768861 93.174777 48.313914 24.243317 1.794380 17.064428 1.744345 58.167923 6.104181 2007.769705
std 421.321334 42.746880 22.376841 4955.517327 1.436812 1.113740 30.390071 21.130467 177.625900 455.268042 ... 217.048611 127.744882 68.883364 67.227765 20.207842 56.609763 30.491646 630.806978 2.722432 1.301740
min 1461.000000 20.000000 21.000000 1470.000000 1.000000 1.000000 1879.000000 1950.000000 0.000000 0.000000 ... 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 1.000000 2006.000000
25% 1825.500000 20.000000 58.000000 7391.000000 5.000000 5.000000 1953.000000 1963.000000 0.000000 0.000000 ... 318.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 4.000000 2007.000000
50% 2190.000000 50.000000 67.000000 9399.000000 6.000000 5.000000 1973.000000 1992.000000 0.000000 350.500000 ... 480.000000 0.000000 28.000000 0.000000 0.000000 0.000000 0.000000 0.000000 6.000000 2008.000000
75% 2554.500000 70.000000 80.000000 11517.500000 7.000000 6.000000 2001.000000 2004.000000 164.000000 753.500000 ... 576.000000 168.000000 72.000000 0.000000 0.000000 0.000000 0.000000 0.000000 8.000000 2009.000000
max 2919.000000 190.000000 200.000000 56600.000000 10.000000 9.000000 2010.000000 2010.000000 1290.000000 4010.000000 ... 1488.000000 1424.000000 742.000000 1012.000000 360.000000 576.000000 800.000000 17000.000000 12.000000 2010.000000

8 rows × 37 columns

Total number of quantitative and qualitative variables

In [401]:
quantative=[f for f in train_df.columns if train_df.dtypes[f]!=object]
print("total number of quantative varibale is", len(quantative)) 
print(quantative)
total number of quantative varibale is 38
['Id', 'MSSubClass', 'LotFrontage', 'LotArea', 'OverallQual', 'OverallCond', 'YearBuilt', 'YearRemodAdd', 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF', 'LowQualFinSF', 'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr', 'TotRmsAbvGrd', 'Fireplaces', 'GarageYrBlt', 'GarageCars', 'GarageArea', 'WoodDeckSF', 'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea', 'MiscVal', 'MoSold', 'YrSold', 'SalePrice']
In [402]:
qualitative=[f for f in train_df.columns if train_df.dtypes[f]==object]
print("total number of qualitative variable is", len(qualitative))
print(qualitative)
total number of qualitative variable is 43
['MSZoning', 'Street', 'Alley', 'LotShape', 'LandContour', 'Utilities', 'LotConfig', 'LandSlope', 'Neighborhood', 'Condition1', 'Condition2', 'BldgType', 'HouseStyle', 'RoofStyle', 'RoofMatl', 'Exterior1st', 'Exterior2nd', 'MasVnrType', 'ExterQual', 'ExterCond', 'Foundation', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'Heating', 'HeatingQC', 'CentralAir', 'Electrical', 'KitchenQual', 'Functional', 'FireplaceQu', 'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond', 'PavedDrive', 'PoolQC', 'Fence', 'MiscFeature', 'SaleType', 'SaleCondition']
In [403]:
#Now look at Output variable(SalesPrice)
train_df['SalePrice'].describe()
Out[403]:
count      1460.000000
mean     180921.195890
std       79442.502883
min       34900.000000
25%      129975.000000
50%      163000.000000
75%      214000.000000
max      755000.000000
Name: SalePrice, dtype: float64

correlation maping

In [404]:
corrmat=train_df.corr()
plt.subplots(figsize=(30, 20))
sns.heatmap(corrmat, annot=True,cmap=plt.cm.Reds);