개발하고 싶어요
베이지안 최적화 기반 본문
베이지안 최적화 기반의 HyperOpt를 이용한 하이퍼 파라미터 튜닝¶
HyperOpt 사용하기¶
In [21]:
# !pip install hyperopt
In [1]:
# 입력 변수명과 입력값의 검색 공간을 설정
from hyperopt import hp
# -10 ~ 10까지 1간격을 가지는 입력 변수 x와 -15 ~ 15까지 1간격으로 입력 변수 y 설정
search_space = {'x' : hp.quniform('x', -10, 10, 1), 'y' : hp.quniform('y', -15, 15, 1)}
In [2]:
# 목적 함수 생성
# 변숫값과 검색 공간을 가지는 딕셔너리를 인자로 받고, 특정 값을 반환하는 구조
from hyperopt import STATUS_OK
# 목적 함수를 생성. 변숫값과 변수 검색 공간을 가지는 딕셔너리를 인자로 받고, 특정 값을 반환
def objective_func(search_space):
x = search_space['x']
y = search_space['y']
retval = x ** 2 - 20 * y
return retval
In [4]:
# 입력값의 검색 공간과 목적 함수를 설정했으면 목적 함수의 반환값이 최소화 될 수 있는 최적의 입력값을 베이지안 최적화 기법에 기반하여 찾기
import numpy as np
from hyperopt import fmin, tpe, Trials
# 입력 결괏값을 지정한 Trials 객체값 생성
trial_val = Trials()
# 목적 함수의 최솟값을 반환하는 최적 입력 변숫값을 5번의 입력값 시도(max_evals = 5)로 찾기
best_01 = fmin(fn = objective_func, space = search_space, algo = tpe.suggest, max_evals = 5, trials = trial_val, rstate = np.random.default_rng(seed = 0))
print('best : ', best_01)
100%|█████████████████████████████████████████████████████████████| 5/5 [00:00<00:00, 997.93trial/s, best loss: -224.0]
best : {'x': -4.0, 'y': 12.0}
=> x는 0에 가까울수록 y는 15에 가까울수록 반환값이 최소로 근사될 수 있다
In [5]:
trial_val = Trials()
# max_evals를 20회로 늘려서
best_02 = fmin(fn = objective_func, space = search_space, algo = tpe.suggest, max_evals = 20, trials = trial_val, rstate = np.random.default_rng(seed = 0))
print('best : ', best_02)
100%|██████████████████████████████████████████████████████████| 20/20 [00:00<00:00, 1056.01trial/s, best loss: -296.0]
best : {'x': 2.0, 'y': 15.0}
=> x는 2로 y는 15로 목적 함수의 최적 최솟값을 근사할 수 있다
In [6]:
# fmin()에 인자로 들어가는 Trials 객체의 result 속성에 파이썬 리스트로 목적 함수 반환값들이 저장
# 리스트 내부의 개별 원소는 {'loss' : 함수 반환값, 'status' : 반환 상태값}와 같은 딕셔너리
print(trial_val.results)
[{'loss': -64.0, 'status': 'ok'}, {'loss': -184.0, 'status': 'ok'}, {'loss': 56.0, 'status': 'ok'}, {'loss': -224.0, 'status': 'ok'}, {'loss': 61.0, 'status': 'ok'}, {'loss': -296.0, 'status': 'ok'}, {'loss': -40.0, 'status': 'ok'}, {'loss': 281.0, 'status': 'ok'}, {'loss': 64.0, 'status': 'ok'}, {'loss': 100.0, 'status': 'ok'}, {'loss': 60.0, 'status': 'ok'}, {'loss': -39.0, 'status': 'ok'}, {'loss': 1.0, 'status': 'ok'}, {'loss': -164.0, 'status': 'ok'}, {'loss': 21.0, 'status': 'ok'}, {'loss': -56.0, 'status': 'ok'}, {'loss': 284.0, 'status': 'ok'}, {'loss': 176.0, 'status': 'ok'}, {'loss': -171.0, 'status': 'ok'}, {'loss': 0.0, 'status': 'ok'}]
In [7]:
# Trials 객체의 vals 속성에 {'입력변수명' : 개별 수행 시마다 입력된 값 리스트}형태로 저장
print(trial_val.vals)
{'x': [-6.0, -4.0, 4.0, -4.0, 9.0, 2.0, 10.0, -9.0, -8.0, -0.0, -0.0, 1.0, 9.0, 6.0, 9.0, 2.0, -2.0, -4.0, 7.0, -0.0], 'y': [5.0, 10.0, -2.0, 12.0, 1.0, 15.0, 7.0, -10.0, 0.0, -5.0, -3.0, 2.0, 4.0, 10.0, 3.0, 3.0, -14.0, -8.0, 11.0, -0.0]}
In [8]:
# results와 vals 속성값들을 dataframe로 확인
import pandas as pd
# results에서 loss 키값에 해당하는 밸류들을 추출하여 list 생성
losses = [loss_dict['loss'] for loss_dict in trial_val.results]
# dataframe로 생성
result_df = pd.DataFrame({'x' : trial_val.vals['x'], 'y' : trial_val.vals['y'], 'losses' : losses})
result_df
Out[8]:
| x | y | losses | |
|---|---|---|---|
| 0 | -6.0 | 5.0 | -64.0 |
| 1 | -4.0 | 10.0 | -184.0 |
| 2 | 4.0 | -2.0 | 56.0 |
| 3 | -4.0 | 12.0 | -224.0 |
| 4 | 9.0 | 1.0 | 61.0 |
| 5 | 2.0 | 15.0 | -296.0 |
| 6 | 10.0 | 7.0 | -40.0 |
| 7 | -9.0 | -10.0 | 281.0 |
| 8 | -8.0 | 0.0 | 64.0 |
| 9 | -0.0 | -5.0 | 100.0 |
| 10 | -0.0 | -3.0 | 60.0 |
| 11 | 1.0 | 2.0 | -39.0 |
| 12 | 9.0 | 4.0 | 1.0 |
| 13 | 6.0 | 10.0 | -164.0 |
| 14 | 9.0 | 3.0 | 21.0 |
| 15 | 2.0 | 3.0 | -56.0 |
| 16 | -2.0 | -14.0 | 284.0 |
| 17 | -4.0 | -8.0 | 176.0 |
| 18 | 7.0 | 11.0 | -171.0 |
| 19 | -0.0 | -0.0 | 0.0 |
HyperOpt를 이용한 XGBoost 하이퍼 파라미터 최적화¶
위스콘신 유방암 데이터 세트를 다시 로딩하여 학습, 검증, 테스트 데이터로 분리
In [10]:
import pandas as pd
import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
import warnings
warnings.filterwarnings('ignore')
dataset = load_breast_cancer() # 데이터 불러오기
dataset.keys()
df = pd.DataFrame(data = dataset.data, columns = dataset.feature_names) # dataframe로 변경
df['target'] = dataset.target # 타겟 값 추가
X_features = df.iloc[:, :-1]
y_label = df.iloc[:, -1]
# 전체 데이터 중 80%는 학습용 데이터, 20%는 테스트용 데이터 추출
X_train, X_test, y_train, y_test = train_test_split(X_features, y_label, test_size = 0.2, random_state = 156)
# 앞에서 추출한 학습 데이터를 다시 학습과 검증 데이터로 분리
X_tr, X_val, y_tr, y_val = train_test_split(X_train, y_train, test_size = 0.1, random_state = 156)
Out[10]:
dict_keys(['data', 'target', 'frame', 'target_names', 'DESCR', 'feature_names', 'filename', 'data_module'])In [12]:
# 하이퍼 파라미터 검색 공간을 설정
from hyperopt import hp
# max_depth는 5에서 20까지 1간격으로, min_child_weight는 1에서 2까지 1 간격으로
# colsample_bytree는 0.5에서 1사이, learning_rate는 0.01에서 0.2 사이 정규 분포된 값으로 검색
xgb_search_space = {'max_depth' : hp.quniform('max_depth', 5, 20, 1),
'min_child_weight' : hp.quniform('min_child_weight', 1, 2, 1),
'colsample_bytree' : hp.uniform('colsample_bytree', 0.5, 1),
'learning_rate' : hp.uniform('learning_rate', 0.01, 0.2)}
In [16]:
# 다음으로는 목적 함수 설정
# 검색 공간에서 목적 함수로 입력되는 모든 인자들은 실수형 값
# HyperOpt의 목적 함수는 최솟값을 반환할 수 있도록 최적화해야 하기 때문에 정확도와 같이 값이 클수록 좋은 성능 지표일 경우 -1을 곱한뒤 반환
from sklearn.model_selection import cross_val_score
from xgboost import XGBClassifier
from hyperopt import STATUS_OK
# fmin()에서 입력된 search_space값으로 입력된 모든 값은 실수형
# XGBClassifier의 정수형 하이퍼 파라미터는 정수형 변환을 해줘야 한다
# 정확도는 높을수록 더 좋은 수치이다. -1 * 정확도를 곱해서 큰 정확도 값일수록 최소가 되도록 변환
def objective_func(search_space):
# 수행 시간 절약을 위해 nestimators는 100으로 축소
xgb = XGBClassifier(n_estimators = 100, max_depth = int(search_space['max_depth']),
min_child_weight = int(search_space['min_child_weight']),
colsample_bytree = search_space['colsample_bytree'],
learning_rate = search_space['learning_rate'],
eval_metric = 'logloss')
accuracy = cross_val_score(xgb, X_train, y_train, scoring = 'accuracy', cv = 3)
# accuracy는 cv = 3개수만큼 roc_auc 결과를 리스트로 가짐. 이를 평균해서 반환하되 -1을 곱함
return {'loss' : -1 * np.mean(accuracy), 'status' : STATUS_OK}
In [17]:
# fmin()을 이요해 최적 하이퍼 파라미터 도출
from hyperopt import fmin, tpe, Trials
trial_val = Trials()
best = fmin(fn = objective_func,
space = xgb_search_space,
algo = tpe.suggest,
max_evals = 50, # 최대 반복 횟수 지정
trials = trial_val,
rstate = np.random.default_rng(seed = 9))
print('best : ', best)
100%|███████████████████████████████████████████████| 50/50 [00:06<00:00, 7.23trial/s, best loss: -0.9670616939700244]
best : {'colsample_bytree': 0.5424149213362504, 'learning_rate': 0.12601372924444681, 'max_depth': 17.0, 'min_child_weight': 2.0}
In [19]:
# fmin()으로 추출된 최적 하이퍼 파라미터를 직접 XGBClassifier에 인자로 입력하기 전 정수형 하이퍼 파라미터는 정수형으로 형 변환을, 실수형 하이퍼 파라미터는 소수점 5자리까지만 변환 후 확인
print('colsample_bytree{}, learning_rate : {}, max_depth : {}, min_child_weight : {}'.format(
round(best['colsample_bytree'], 5), round(best['learning_rate'],n 5),
int(best['max_depth']), int(best['min_child_weight'])))
colsample_bytree0.54241, learning_rate : 0.12601, max_depth : 17, min_child_weight : 2
In [22]:
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score
def get_clf_eval(y_test, pred = None, pred_proba = None):
confusion = confusion_matrix(y_test, pred)
accuracy = accuracy_score(y_test, pred)
precision = precision_score(y_test, pred)
recall = recall_score(y_test , pred)
f1 = f1_score(y_test, pred)
# ROC-AUC 추가
roc_auc = roc_auc_score(y_test, pred_proba)
print('오차행렬')
print(confusion)
# ROC-AUC 출력
print('정확도 : {0:.4f}, 정밀도 : {1:.4f}, 재현율 : {2:.4f}, F1 : {3:.4f}, AUC : {4:.4f}'.format(accuracy, precision, recall, f1, roc_auc))
In [24]:
# 도출된 최적 하이퍼 파라미터들을 이용해 XGBClassifier를 재학습 후 성능 평가 결과 확인
xgb_wrapper = XGBClassifier(n_estimators = 400,
leaning_rate = round(best['learning_rate'], 5),
max_depth = int(best['max_depth']),
min_child_weight = int(best['min_child_weight']),
colsample_bytree = round(best['colsample_bytree'], 5))
evals = [(X_tr, y_tr), (X_val, y_val)]
xgb_wrapper.fit(X_tr, y_tr, early_stopping_rounds = 50, eval_metric = 'logloss',
eval_set = evals, verbose = True)
pred = xgb_wrapper.predict(X_test)
pred_proba = xgb_wrapper.predict_proba(X_test)[:, 1]
[14:58:58] WARNING: C:/buildkite-agent/builds/buildkite-windows-cpu-autoscaling-group-i-08de971ced8a8cdc6-1/xgboost/xgboost-ci-windows/src/learner.cc:767:
Parameters: { "leaning_rate" } are not used.
[0] validation_0-logloss:0.46780 validation_1-logloss:0.53951
[1] validation_0-logloss:0.33860 validation_1-logloss:0.45055
[2] validation_0-logloss:0.25480 validation_1-logloss:0.38982
[3] validation_0-logloss:0.19908 validation_1-logloss:0.36525
[4] validation_0-logloss:0.15836 validation_1-logloss:0.34947
[5] validation_0-logloss:0.12936 validation_1-logloss:0.33216
[6] validation_0-logloss:0.10800 validation_1-logloss:0.32261
[7] validation_0-logloss:0.09188 validation_1-logloss:0.31803
[8] validation_0-logloss:0.07968 validation_1-logloss:0.31458
[9] validation_0-logloss:0.06982 validation_1-logloss:0.29838
[10] validation_0-logloss:0.06111 validation_1-logloss:0.29127
[11] validation_0-logloss:0.05569 validation_1-logloss:0.29192
[12] validation_0-logloss:0.04953 validation_1-logloss:0.29192
[13] validation_0-logloss:0.04482 validation_1-logloss:0.28254
[14] validation_0-logloss:0.04086 validation_1-logloss:0.28237
[15] validation_0-logloss:0.03751 validation_1-logloss:0.28031
[16] validation_0-logloss:0.03485 validation_1-logloss:0.26671
[17] validation_0-logloss:0.03265 validation_1-logloss:0.26695
[18] validation_0-logloss:0.03086 validation_1-logloss:0.26435
[19] validation_0-logloss:0.02923 validation_1-logloss:0.26792
[20] validation_0-logloss:0.02784 validation_1-logloss:0.26543
[21] validation_0-logloss:0.02651 validation_1-logloss:0.26652
[22] validation_0-logloss:0.02606 validation_1-logloss:0.26353
[23] validation_0-logloss:0.02520 validation_1-logloss:0.26226
[24] validation_0-logloss:0.02427 validation_1-logloss:0.25847
[25] validation_0-logloss:0.02377 validation_1-logloss:0.26393
[26] validation_0-logloss:0.02296 validation_1-logloss:0.26746
[27] validation_0-logloss:0.02264 validation_1-logloss:0.26769
[28] validation_0-logloss:0.02236 validation_1-logloss:0.27243
[29] validation_0-logloss:0.02116 validation_1-logloss:0.26105
[30] validation_0-logloss:0.02091 validation_1-logloss:0.26321
[31] validation_0-logloss:0.02065 validation_1-logloss:0.25900
[32] validation_0-logloss:0.02042 validation_1-logloss:0.25218
[33] validation_0-logloss:0.02014 validation_1-logloss:0.25071
[34] validation_0-logloss:0.01987 validation_1-logloss:0.25543
[35] validation_0-logloss:0.01962 validation_1-logloss:0.25458
[36] validation_0-logloss:0.01940 validation_1-logloss:0.25232
[37] validation_0-logloss:0.01917 validation_1-logloss:0.25137
[38] validation_0-logloss:0.01891 validation_1-logloss:0.25232
[39] validation_0-logloss:0.01872 validation_1-logloss:0.25478
[40] validation_0-logloss:0.01852 validation_1-logloss:0.24843
[41] validation_0-logloss:0.01835 validation_1-logloss:0.25245
[42] validation_0-logloss:0.01819 validation_1-logloss:0.24804
[43] validation_0-logloss:0.01800 validation_1-logloss:0.24821
[44] validation_0-logloss:0.01782 validation_1-logloss:0.24606
[45] validation_0-logloss:0.01764 validation_1-logloss:0.24532
[46] validation_0-logloss:0.01749 validation_1-logloss:0.24621
[47] validation_0-logloss:0.01736 validation_1-logloss:0.24575
[48] validation_0-logloss:0.01722 validation_1-logloss:0.24590
[49] validation_0-logloss:0.01708 validation_1-logloss:0.24403
[50] validation_0-logloss:0.01695 validation_1-logloss:0.24420
[51] validation_0-logloss:0.01679 validation_1-logloss:0.24644
[52] validation_0-logloss:0.01668 validation_1-logloss:0.24758
[53] validation_0-logloss:0.01652 validation_1-logloss:0.24236
[54] validation_0-logloss:0.01640 validation_1-logloss:0.23969
[55] validation_0-logloss:0.01630 validation_1-logloss:0.23905
[56] validation_0-logloss:0.01619 validation_1-logloss:0.23847
[57] validation_0-logloss:0.01607 validation_1-logloss:0.23958
[58] validation_0-logloss:0.01594 validation_1-logloss:0.24174
[59] validation_0-logloss:0.01584 validation_1-logloss:0.24002
[60] validation_0-logloss:0.01573 validation_1-logloss:0.23589
[61] validation_0-logloss:0.01561 validation_1-logloss:0.23594
[62] validation_0-logloss:0.01552 validation_1-logloss:0.23950
[63] validation_0-logloss:0.01542 validation_1-logloss:0.23957
[64] validation_0-logloss:0.01532 validation_1-logloss:0.23573
[65] validation_0-logloss:0.01524 validation_1-logloss:0.23897
[66] validation_0-logloss:0.01515 validation_1-logloss:0.23894
[67] validation_0-logloss:0.01507 validation_1-logloss:0.23711
[68] validation_0-logloss:0.01496 validation_1-logloss:0.23724
[69] validation_0-logloss:0.01488 validation_1-logloss:0.23623
[70] validation_0-logloss:0.01482 validation_1-logloss:0.23321
[71] validation_0-logloss:0.01473 validation_1-logloss:0.23709
[72] validation_0-logloss:0.01465 validation_1-logloss:0.23816
[73] validation_0-logloss:0.01458 validation_1-logloss:0.23679
[74] validation_0-logloss:0.01452 validation_1-logloss:0.23688
[75] validation_0-logloss:0.01444 validation_1-logloss:0.23684
[76] validation_0-logloss:0.01437 validation_1-logloss:0.23980
[77] validation_0-logloss:0.01432 validation_1-logloss:0.23685
[78] validation_0-logloss:0.01424 validation_1-logloss:0.23752
[79] validation_0-logloss:0.01418 validation_1-logloss:0.23639
[80] validation_0-logloss:0.01412 validation_1-logloss:0.23636
[81] validation_0-logloss:0.01406 validation_1-logloss:0.23700
[82] validation_0-logloss:0.01401 validation_1-logloss:0.23555
[83] validation_0-logloss:0.01396 validation_1-logloss:0.23566
[84] validation_0-logloss:0.01391 validation_1-logloss:0.23430
[85] validation_0-logloss:0.01385 validation_1-logloss:0.23662
[86] validation_0-logloss:0.01379 validation_1-logloss:0.23934
[87] validation_0-logloss:0.01375 validation_1-logloss:0.23858
[88] validation_0-logloss:0.01370 validation_1-logloss:0.23759
[89] validation_0-logloss:0.01364 validation_1-logloss:0.23757
[90] validation_0-logloss:0.01358 validation_1-logloss:0.23869
[91] validation_0-logloss:0.01353 validation_1-logloss:0.23930
[92] validation_0-logloss:0.01349 validation_1-logloss:0.23792
[93] validation_0-logloss:0.01344 validation_1-logloss:0.23789
[94] validation_0-logloss:0.01339 validation_1-logloss:0.23693
[95] validation_0-logloss:0.01335 validation_1-logloss:0.23936
[96] validation_0-logloss:0.01331 validation_1-logloss:0.23997
[97] validation_0-logloss:0.01326 validation_1-logloss:0.23996
[98] validation_0-logloss:0.01322 validation_1-logloss:0.23865
[99] validation_0-logloss:0.01318 validation_1-logloss:0.23809
[100] validation_0-logloss:0.01314 validation_1-logloss:0.23908
[101] validation_0-logloss:0.01311 validation_1-logloss:0.23965
[102] validation_0-logloss:0.01307 validation_1-logloss:0.23735
[103] validation_0-logloss:0.01303 validation_1-logloss:0.23652
[104] validation_0-logloss:0.01300 validation_1-logloss:0.23871
[105] validation_0-logloss:0.01297 validation_1-logloss:0.23818
[106] validation_0-logloss:0.01294 validation_1-logloss:0.23810
[107] validation_0-logloss:0.01291 validation_1-logloss:0.23868
[108] validation_0-logloss:0.01288 validation_1-logloss:0.23957
[109] validation_0-logloss:0.01286 validation_1-logloss:0.23903
[110] validation_0-logloss:0.01283 validation_1-logloss:0.23806
[111] validation_0-logloss:0.01281 validation_1-logloss:0.23858
[112] validation_0-logloss:0.01279 validation_1-logloss:0.23779
[113] validation_0-logloss:0.01276 validation_1-logloss:0.23971
[114] validation_0-logloss:0.01274 validation_1-logloss:0.23891
[115] validation_0-logloss:0.01272 validation_1-logloss:0.23843
[116] validation_0-logloss:0.01270 validation_1-logloss:0.23919
[117] validation_0-logloss:0.01268 validation_1-logloss:0.23903
[118] validation_0-logloss:0.01266 validation_1-logloss:0.23950
[119] validation_0-logloss:0.01264 validation_1-logloss:0.23906
[120] validation_0-logloss:0.01262 validation_1-logloss:0.23827
Out[24]:
XGBClassifier(base_score=None, booster=None, callbacks=None,
colsample_bylevel=None, colsample_bynode=None,
colsample_bytree=0.54241, early_stopping_rounds=None,
enable_categorical=False, eval_metric=None, feature_types=None,
gamma=None, gpu_id=None, grow_policy=None, importance_type=None,
interaction_constraints=None, leaning_rate=0.12601,
learning_rate=None, max_bin=None, max_cat_threshold=None,
max_cat_to_onehot=None, max_delta_step=None, max_depth=17,
max_leaves=None, min_child_weight=2, missing=nan,
monotone_constraints=None, n_estimators=400, n_jobs=None,
num_parallel_tree=None, predictor=None, ...)On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
XGBClassifier(base_score=None, booster=None, callbacks=None,
colsample_bylevel=None, colsample_bynode=None,
colsample_bytree=0.54241, early_stopping_rounds=None,
enable_categorical=False, eval_metric=None, feature_types=None,
gamma=None, gpu_id=None, grow_policy=None, importance_type=None,
interaction_constraints=None, leaning_rate=0.12601,
learning_rate=None, max_bin=None, max_cat_threshold=None,
max_cat_to_onehot=None, max_delta_step=None, max_depth=17,
max_leaves=None, min_child_weight=2, missing=nan,
monotone_constraints=None, n_estimators=400, n_jobs=None,
num_parallel_tree=None, predictor=None, ...)In [25]:
get_clf_eval(y_test, pred, pred_proba)
오차행렬
[[33 4]
[ 3 74]]
정확도 : 0.9386, 정밀도 : 0.9487, 재현율 : 0.9610, F1 : 0.9548, AUC : 0.9933
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
'ML' 카테고리의 다른 글
| 경사하강법 (0) | 2024.01.29 |
|---|---|
| 보스턴 주택 가격 예측 (0) | 2024.01.29 |
| lightgbm_분류 (0) | 2024.01.23 |
| 랜덤포레스트_분류 (0) | 2024.01.23 |
| xgboost_분류 (0) | 2024.01.23 |