2. 머신러닝 프로젝트의 처음부터 끝까지¶
. 머신러닝 주택 회사에 오신 것을 환영합니다! 여러분이 해야 할 일은 캘리포니아 인구조사 데이터를 사용해 이 지역의 주택 가격 모델을 만드는 것입니다.¶
2.0 설정¶
In [1]:
# 공통
from warnings import simplefilter
import numpy as np
import os
import pandas as pd
import sklearn.linear_model
import matplotlib.pyplot as plt
from matplotlib import font_manager, rc
from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedShuffleSplit
from pandas.plotting import scatter_matrix
from sklearn.preprocessing import OneHotEncoder
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.impute import SimpleImputer
# from sklearn.preprocessing import Imputer
from sklearn.pipeline import FeatureUnion
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error
from sklearn.tree import DecisionTreeRegressor
from sklearn.model_selection import cross_val_score
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import GridSearchCV
simplefilter(action='ignore', category=FutureWarning)
# 일관된 출력을 위해 유사난수 초기화
np.random.seed(42)
%matplotlib inline
font_name = font_manager.FontProperties(fname="c:/Windows/Fonts/malgun.ttf").get_name()
rc('font', family=font_name)
plt.rcParams['axes.labelsize'] = 14
plt.rcParams['xtick.labelsize'] = 12
plt.rcParams['ytick.labelsize'] = 12
plt.rcParams['axes.unicode_minus'] = False
# 그림을 저장할 폴드
PROJECT_ROOT_DIR = "C:/Users/Admin/Desktop/ML/"
# PROJECT_ROOT_DIR = "C:/Users/User/Desktop/ML/"
# PROJECT_ROOT_DIR = "C:/Users/sally/Dropbox/2019-Fall-Semester/ML"
CHAPTER_ID = "end_to_end_project"
IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, "images", CHAPTER_ID)
def save_fig(fig_id, tight_layout=True):
path = os.path.join(IMAGES_PATH, fig_id + ".png")
if tight_layout:
plt.tight_layout()
plt.savefig(path, format='png', dpi=300)
2.3.2 데이터 다운로드¶
In [2]:
housing_path = os.path.join("datasets","housing","")
In [3]:
housing = pd.read_csv(housing_path + "housing.csv")
2.3.3 데이터 훑어보기¶
In [4]:
housing.head()
Out[4]:
. 207개의 구역에서 total_bedrooms가 결측¶
In [5]:
housing.info()
. ocean_proximity 특성의 도수분포표 작성¶
. pandas.Series.value_count https://pandas.pydata.org/pandas-docs/stable/generated/pandas.Series.value_counts.html¶
In [6]:
housing["ocean_proximity"].value_counts()
Out[6]:
. 숫자형 특성의 summary statistics¶
In [7]:
housing.describe()
Out[7]:
In [8]:
housing.hist(bins=50, figsize=(20,15))
save_fig("attribute_histogram_plots")
2.3.4 테스트 세트 만들기¶
. 단순임의추출법(simple random sampling; SRS)¶
. sklearn.model_selection.train_test_split¶
Split arrays or matrices into random train and test subsets
https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.train_test_split.html
In [9]:
train_set, test_set = train_test_split(housing, test_size=0.2, random_state=42)
In [10]:
test_set.head()
Out[10]:
In [11]:
# 소득 카테고리 개수를 제한하기 위해 1.5로 나눕니다.
housing["income_cat"] = np.ceil(housing["median_income"] / 1.5)
In [12]:
housing[["median_income", "income_cat"]]
Out[12]:
. pandas.DataFrame.where¶
https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.where.html
In [13]:
# 5 이상은 5로 레이블합니다.
housing["income_cat"].where(housing["income_cat"] < 5, 5.0, inplace=True)
In [14]:
housing[["median_income", "income_cat"]]
Out[14]:
In [15]:
housing["income_cat"].value_counts()
Out[15]:
In [16]:
housing["income_cat"].hist()
save_fig('income_category_hist')
In [17]:
split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)
In [18]:
for train_index, test_index in split.split(housing, housing["income_cat"]):
print(train_index)
strat_train_set = housing.loc[train_index]
strat_test_set = housing.loc[test_index]
In [19]:
strat_train_set.head()
Out[19]:
In [20]:
strat_test_set["income_cat"].value_counts(normalize=True)
Out[20]:
In [21]:
def income_cat_proportions(data):
return data["income_cat"].value_counts(normalize=True)
In [22]:
train_set, test_set = train_test_split(housing, test_size=0.2, random_state=42)
In [23]:
compare_props = pd.DataFrame({
"Overall": income_cat_proportions(housing),
"Stratified": income_cat_proportions(strat_test_set),
"Random": income_cat_proportions(test_set),
}).sort_index()
In [24]:
compare_props
Out[24]:
In [25]:
compare_props["Rand. %error"] = 100 * (compare_props["Random"] - compare_props["Overall"]) / compare_props["Overall"]
In [26]:
compare_props["Strat. %error"] = 100 * (compare_props["Stratified"] - compare_props["Overall"]) / compare_props["Overall"]
In [27]:
compare_props
Out[27]:
. pandas.DataFrame.drop¶
Drop specified labels from rows or columns.
https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.drop.html
In [28]:
for set_ in (strat_train_set, strat_test_set):
set_.drop("income_cat", axis=1, inplace=True)
In [29]:
strat_train_set.info()
2.4 데이터 이해를 위한 탐색과 시각화¶
In [30]:
housing = strat_train_set.copy()
. pandas.DataFrame.plot¶
https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.plot.html
In [31]:
housing.plot(kind="scatter", x="longitude", y="latitude")
save_fig("bad_visualization_plot")
. 원의 반지름은 구역의 인구에 비례, 색깔은 중간 주택 가격 (파란색(낮은 가격)에서 빨간색(높은 가격)으로 변하는 jet), 투명도는 0.4로 지정¶
In [32]:
housing.plot(kind="scatter", x="longitude", y="latitude", alpha=0.4,
s=housing["population"]/100,label="population", figsize=(10,7),
c="median_house_value", cmap=plt.get_cmap("jet"), colorbar=True, sharex=False)
save_fig("better_visualization_plot")
In [33]:
corr_matrix = housing.corr()
corr_matrix
Out[33]:
In [34]:
corr_matrix["median_house_value"].sort_values(ascending=False)
Out[34]:
In [35]:
attributes = ["median_house_value", "median_income", "total_rooms",
"housing_median_age"]
scatter_matrix(housing[attributes], figsize=(12, 8))
save_fig("scatter_matrix_plot")
In [36]:
housing.plot(kind="scatter", x="median_income", y="median_house_value",
alpha=0.1)
save_fig("income_vs_house_value_scatterplot")
2.4.3 특성 조합으로 실험¶
In [37]:
housing["rooms_per_household"] = housing["total_rooms"]/housing["households"]
In [38]:
housing["bedrooms_per_room"] = housing["total_bedrooms"]/housing["total_rooms"]
In [39]:
housing["population_per_household"] = housing["population"]/housing["households"]
In [40]:
corr_matrix = housing.corr()
corr_matrix["median_house_value"].sort_values(ascending=False)
Out[40]:
In [41]:
housing.plot(kind="scatter", x="rooms_per_household", y="median_house_value",
alpha=0.2)
# plt.axis([0, 5, 0, 520000])
plt.show()
In [42]:
housing.describe()
Out[42]:
2.5 머신러닝 알고리즘을 위한 데이터 준비¶
In [43]:
housing = strat_train_set.drop("median_house_value", axis=1) # 훈련 세트를 위해 레이블 삭제
In [44]:
housing_labels = strat_train_set["median_house_value"].copy()
2.5.1 데이터 정제¶
. pandas.DataFrame.any¶
https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.any.html
. pandas.Datafeame.dropna¶
Remove missing values.
https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.dropna.html
. pandas.DataFrame.fillna¶
Fill NA/NaN values using the specified method
https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.fillna.html
In [45]:
sample_incomplete_rows = housing[housing.isnull().any(axis=1)].head()
sample_incomplete_rows
Out[45]:
In [46]:
sample_incomplete_rows.dropna(subset=["total_bedrooms"]) # 옵션 1: 해당 구역을 제거
# sample_incomplete_rows.dropna() # 옵션 1: 해당 구역을 제거
Out[46]:
In [47]:
sample_incomplete_rows.drop("total_bedrooms", axis=1) # 옵션 2: 전체 특성을 삭제
Out[47]:
In [48]:
median = housing["total_bedrooms"].median()
In [49]:
sample_incomplete_rows["total_bedrooms"].fillna(median, inplace=True) # 옵션 3: median으로 채움
sample_incomplete_rows
Out[49]:
. sklearn.preprocessing.Imputer¶
http://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.Imputer.html
In [50]:
imputer = SimpleImputer(strategy="median")
. median이 수치형 특성에서만 계산될 수 있기 때문에 텍스트 특성을 삭제¶
In [51]:
housing_num = housing.drop('ocean_proximity', axis = 1)
In [52]:
imputer.fit(housing_num)
Out[52]:
In [53]:
imputer.statistics_
Out[53]:
. 각 특성의 median이 수동으로 계산한 것과 같은지 확인¶
. pandas.DataFrame.median¶
https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.median.html
. pandas.Series.values¶
https://pandas.pydata.org/pandas-docs/stable/generated/pandas.Series.values.html
In [54]:
housing_num.median().values
Out[54]:
. 훈련 세트 변환¶
In [55]:
X = imputer.transform(housing_num) #변환된 특성들이 들아 있는 numpy 배열
X
Out[55]:
In [56]:
housing_tr = pd.DataFrame(X, columns=housing_num.columns,
index = list(housing.index.values))
. pandas.DataFrame.loc¶
https://pandas.pydata.org/pandas-docs/version/0.23/generated/pandas.DataFrame.loc.html
In [57]:
housing_tr.loc[sample_incomplete_rows.index.values]
Out[57]:
2.5.2 텍스트와 범주형 특성 다루기¶
. 범주형 입력 특성인 ocean_proximity를 전처리¶
In [58]:
housing_cat = housing['ocean_proximity']
housing_cat.head(10)
Out[58]:
. pandas의 factorize() 메서드는 문자열 범주형 특성을 머신러닝 알고리즘이 다루기 쉬운 숫자 범주형 특성으로 변환¶
In [59]:
housing_cat_encoded, housing_categories = housing_cat.factorize()
In [60]:
housing_cat_encoded[:10]
Out[60]:
In [61]:
housing_categories
Out[61]:
. OneHotEncoder를 사용하여 범주형 값을 원-핫 벡터로 변경¶
. sklearn.preprocessing.OneHotEncoder¶
http://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.OneHotEncoder.html
In [62]:
encoder = OneHotEncoder(categories = 'auto')
In [63]:
housing_cat_1hot = encoder.fit_transform(housing_cat_encoded.reshape(-1,1)) # 행 차원은 unknown, 열 차원은 1 1
housing_cat_1hot
Out[63]:
. OneHotEncoder는 기본적으로 희소 행렬을 반환. 필요하면 밀집 배열로 변환 가능¶
In [64]:
housing_cat_1hot.toarray()
Out[64]:
In [65]:
housing_cat_reshaped = housing_cat.values.reshape(-1, 1) # 텍스트 카테고리
housing_cat_reshaped
Out[65]:
In [66]:
housing_cat_1hot = encoder.fit_transform(housing_cat_reshaped)
housing_cat_1hot
Out[66]:
In [67]:
housing_cat_1hot.toarray()
Out[67]:
In [68]:
encoder.categories_
Out[68]:
2.5.3 나만의 변환기¶
In [69]:
rooms_ix, bedrooms_ix, population_ix, household_ix = 3, 4, 5, 6
In [70]:
class CombinedAttributesAdder(BaseEstimator, TransformerMixin):
def __init__(self, add_bedrooms_per_room = True): # no *args or **kargs
self.add_bedrooms_per_room = add_bedrooms_per_room
def fit(self, X, y=None):
return self # nothing else to do
def transform(self, X, y=None):
rooms_per_household = X[:, rooms_ix] / X[:, household_ix]
population_per_household = X[:, population_ix] / X[:, household_ix]
if self.add_bedrooms_per_room:
bedrooms_per_room = X[:, bedrooms_ix] / X[:, rooms_ix]
return np.c_[X, rooms_per_household, population_per_household,
bedrooms_per_room]
else:
return np.c_[X, rooms_per_household, population_per_household]
In [71]:
attr_adder = CombinedAttributesAdder(add_bedrooms_per_room = False)
In [72]:
housing_extra_attribs = attr_adder.transform(housing.values)
housing_extra_attribs
Out[72]:
In [73]:
housing_extra_attribs = pd.DataFrame(
housing_extra_attribs,
columns=list(housing.columns)+["rooms_per_household", "population_per_household"])
housing_extra_attribs.head()
Out[73]:
2.5.5 변환 파이프라인¶
. 수치 특성을 전처리하기 위한 파이프라인 작성¶
In [74]:
num_pipeline = Pipeline([
# ('imputer', Imputer(strategy="median")),
('imputer', SimpleImputer(strategy="median")),
('attribs_adder', CombinedAttributesAdder()),
('std_scaler', StandardScaler()),
])
In [75]:
housing_num_tr = num_pipeline.fit_transform(housing_num)
In [76]:
housing_num_tr
Out[76]:
. pandas DataFrame 컬럼의 일부를 선택하는 변환기를 작성¶
In [77]:
class DataFrameSelector(BaseEstimator, TransformerMixin):
def __init__(self, attribute_names):
self.attribute_names = attribute_names
def fit(self, X, y=None):
return self
def transform(self, X):
return X[self.attribute_names].values
. 하나의 큰 파이프라인에 이들을 모두 결합하여 수치형과 범주형 특성을 전처리¶
In [78]:
df = pd.DataFrame({'age': [ 3, 29],
'height': [94, 170],
'weight': [31, 115]})
In [79]:
type(df.values)
Out[79]:
In [80]:
num_attribs = list(housing_num)
num_attribs
Out[80]:
In [81]:
cat_attribs = ["ocean_proximity"]
In [82]:
num_pipeline = Pipeline([
('selector', DataFrameSelector(num_attribs)),
('imputer', SimpleImputer(strategy="median")),
# ('imputer', Imputer(strategy="median")),
('attribs_adder', CombinedAttributesAdder()),
('std_scaler', StandardScaler()),
])
In [83]:
cat_pipeline = Pipeline([
('selector', DataFrameSelector(cat_attribs)),
('cat_encoder', encoder),
# ('cat_encoder', OneHotEncoder(categories = 'auto')),
# ('cat_encoder', CategoricalEncoder(encoding="onehot-dense")),
])
In [84]:
full_pipeline = FeatureUnion(transformer_list=[
("num_pipeline", num_pipeline),
("cat_pipeline", cat_pipeline)
])
In [85]:
housing_prepared = full_pipeline.fit_transform(housing)
housing_prepared
Out[85]:
In [86]:
housing_prepared.shape
Out[86]:
In [87]:
housing_prepared.toarray()
Out[87]:
2.6 모델 선택과 훈련¶
2.6.1 훈련 세트에서 훈련하고 평가하기¶
In [88]:
lin_reg = LinearRegression()
lin_reg.fit(housing_prepared, housing_labels) # 선형 회귀 모델 훈련
Out[88]:
. pandas.DataFrame.iloc¶
https://pandas.pydata.org/pandas-docs/version/0.23/generated/pandas.DataFrame.iloc.html
In [89]:
# 훈련 샘플 몇 개를 사용해 전체 파이프라인을 적용
some_data = housing.iloc[:5]
In [90]:
some_labels = housing_labels.iloc[:5]
In [91]:
some_data_prepared = full_pipeline.transform(some_data)
print("예측:", lin_reg.predict(some_data_prepared))
. 실제 값과 비교¶
In [92]:
print("레이블:", list(some_labels))
In [93]:
some_data_prepared
Out[93]:
. 전체 훈련 세트의 mean_square_error 함수 적용¶
In [94]:
housing_predictions = lin_reg.predict(housing_prepared)
In [95]:
lin_mse = mean_squared_error(housing_labels, housing_predictions)
In [96]:
lin_rmse = np.sqrt(lin_mse)
lin_rmse
Out[96]:
In [97]:
tree_reg = DecisionTreeRegressor(random_state=42)
tree_reg.fit(housing_prepared, housing_labels)
Out[97]:
In [98]:
housing_predictions = tree_reg.predict(housing_prepared)
In [99]:
tree_mse = mean_squared_error(housing_labels, housing_predictions)
In [100]:
tree_rmse = np.sqrt(tree_mse)
tree_rmse
Out[100]:
. 과대 적합 모델?¶
2.6.2 교차 검증을 사용한 평가¶
In [101]:
lin_scores = cross_val_score(lin_reg, housing_prepared, housing_labels,
scoring="neg_mean_squared_error", cv=10)
In [102]:
lin_rmse_scores = np.sqrt(-lin_scores)
In [103]:
def display_scores(scores):
print("점수:", scores)
print("평균:", scores.mean())
print("표준편차:", scores.std())
display_scores(lin_rmse_scores)
In [104]:
scores = cross_val_score(tree_reg, housing_prepared, housing_labels,
scoring="neg_mean_squared_error", cv=10)
tree_rmse_scores = np.sqrt(-scores)
display_scores(tree_rmse_scores)
. 앙상블 학습 : 랜덤 포레스트¶
In [105]:
forest_reg = RandomForestRegressor(random_state = 42,n_estimators=10)
forest_reg.fit(housing_prepared, housing_labels)
Out[105]:
In [106]:
housing_predictions = forest_reg.predict(housing_prepared)
In [107]:
forest_mse = mean_squared_error(housing_labels, housing_predictions)
In [108]:
forest_rmse = np.sqrt(forest_mse)
forest_rmse
Out[108]:
. 과대 적합 모델?¶
In [109]:
forest_scores = cross_val_score(forest_reg, housing_prepared, housing_labels,
scoring="neg_mean_squared_error", cv=10)
In [110]:
forest_rmse_scores = np.sqrt(-forest_scores)
In [111]:
display_scores(forest_rmse_scores)
2.7 모델 세부 튜닝¶
2.7.1 그리드 탐색¶
In [112]:
param_grid = [
# 하이퍼파라미터 12(=3×4)개의 조합을 시도합니다.
{'n_estimators': [3, 10, 30], 'max_features': [2, 4, 6, 8]},
# bootstrap은 False로 하고 6(=2×3)개의 조합을 시도합니다.
{'bootstrap': [False], 'n_estimators': [3, 10], 'max_features': [2, 3, 4]},
]
In [113]:
forest_reg = RandomForestRegressor(random_state=42)
# 다섯 폴드에서 훈련하면 총 (12+6)*5=90번의 훈련이 일어납니다.
In [114]:
grid_search = GridSearchCV(forest_reg, param_grid, cv=5, scoring='neg_mean_squared_error',
return_train_score=True, n_jobs=-1)
grid_search.fit(housing_prepared, housing_labels)
Out[114]:
. 최상의 파라미터 조합:¶
In [115]:
grid_search.best_params_
Out[115]:
In [116]:
grid_search.best_estimator_
Out[116]:
. 그리드 탐색에서 테스트한 하이퍼파라미터 조합의 점수를 확인¶
In [117]:
cvres = grid_search.cv_results_
In [118]:
for mean_score, params in zip(cvres["mean_test_score"], cvres["params"]):
print(np.sqrt(-mean_score), params)
2.7.4 최상의 모델과 오차 분석¶
In [119]:
feature_importances = grid_search.best_estimator_.feature_importances_
feature_importances
Out[119]:
In [120]:
extra_attribs = ["rooms_per_hhold", "pop_per_hhold", "bedrooms_per_room"]
In [121]:
# cat_one_hot_attribs = list(cat_encoder.categories_[0])
cat_one_hot_attribs = list(encoder.categories_[0])
cat_one_hot_attribs
Out[121]:
In [122]:
attributes = num_attribs + extra_attribs + cat_one_hot_attribs
In [123]:
sorted(zip(feature_importances, attributes), reverse=True)
Out[123]:
2.7.5 테스트 세트로 시스템 평가하기¶
In [124]:
final_model = grid_search.best_estimator_
In [125]:
X_test = strat_test_set.drop("median_house_value", axis=1)
In [126]:
y_test = strat_test_set["median_house_value"].copy()
In [127]:
X_test_prepared = full_pipeline.transform(X_test)
In [128]:
final_predictions = final_model.predict(X_test_prepared)
In [129]:
final_mse = mean_squared_error(y_test, final_predictions)
In [130]:
final_rmse = np.sqrt(final_mse)
final_rmse
Out[130]: