{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# S4E12 CatBoost Optuna w/count_feature\n","metadata":{"papermill":{"duration":0.014799,"end_time":"2021-06-30T01:42:31.37554","exception":false,"start_time":"2021-06-30T01:42:31.360741","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import lightgbm as lgb\nimport catboost as catb\nimport numpy as np\nimport pandas as pd\nimport random\nimport optuna\nfrom sklearn.model_selection import KFold, train_test_split\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import mean_squared_error","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","papermill":{"duration":2.560292,"end_time":"2021-06-30T01:42:33.94908","exception":false,"start_time":"2021-06-30T01:42:31.388788","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-06T01:05:32.110012Z","iopub.execute_input":"2021-10-06T01:05:32.110354Z","iopub.status.idle":"2021-10-06T01:05:32.115512Z","shell.execute_reply.started":"2021-10-06T01:05:32.110325Z","shell.execute_reply":"2021-10-06T01:05:32.114249Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train0 = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\")\ntest0 = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")","metadata":{"papermill":{"duration":1.678806,"end_time":"2021-06-30T01:42:35.642283","exception":false,"start_time":"2021-06-30T01:42:33.963477","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-06T01:05:32.117202Z","iopub.execute_input":"2021-10-06T01:05:32.117642Z","iopub.status.idle":"2021-10-06T01:05:32.519982Z","shell.execute_reply.started":"2021-10-06T01:05:32.117606Z","shell.execute_reply":"2021-10-06T01:05:32.518953Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df=train0.copy()\nocolumns=[]\nfor c in df.columns:\n    if df[c].dtype=='object': \n        ocolumns+=[c]\nprint(ocolumns)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def _generate_count_feature(series):\n    value_counts = series.value_counts().to_dict() \n    return series.map(value_counts), value_counts\n\ndef create_count_feature_train(input_df):\n    use_columns = ocolumns\n    output_df = pd.DataFrame()\n    mappings = {}  \n    for c in use_columns:\n        x, mapping = _generate_count_feature(input_df[c]) \n        output_df[f\"{c}_count\"] = x\n        mappings[c] = mapping  \n    return output_df, mappings\n\ndef create_count_feature_test(input_df, mappings):\n    use_columns = ocolumns\n    output_df = pd.DataFrame()\n    for c in use_columns:\n        output_df[f\"{c}_count\"] = input_df[c].map(mappings[c]).fillna(0)\n    return output_df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"trainadd,mappings=create_count_feature_train(train0)\ntrain=pd.concat([train0,trainadd],axis=1)\ntestadd=create_count_feature_test(test0,mappings)\ntest=pd.concat([test0,testadd],axis=1)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\ndef labelencoder(df_train, df_test):\n    combined_df = pd.concat([df_train, df_test])\n    for c in combined_df.columns:\n        if combined_df[c].dtype == 'object':\n            combined_df[c] = combined_df[c].fillna('N')\n            lbl = LabelEncoder()\n            lbl.fit(list(combined_df[c].values))\n            combined_df[c] = lbl.transform(combined_df[c].values)\n    return combined_df.iloc[:len(df_train)], combined_df.iloc[len(df_train):]\n\ntrain,test=labelencoder(train,test)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target = train['Premium Amount']\ndata = train.drop(['Premium Amount'],axis=1)\n\ncolumns=data.columns.to_list()\nprint(columns)","metadata":{"papermill":{"duration":0.087259,"end_time":"2021-06-30T01:42:35.884715","exception":false,"start_time":"2021-06-30T01:42:35.797456","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-06T01:05:33.949199Z","iopub.execute_input":"2021-10-06T01:05:33.949662Z","iopub.status.idle":"2021-10-06T01:05:33.961497Z","shell.execute_reply.started":"2021-10-06T01:05:33.949631Z","shell.execute_reply":"2021-10-06T01:05:33.960435Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def objective(trial,data=data,target=target):\n    train_x, test_x, train_y, test_y = train_test_split(data, target, test_size=0.2,random_state=42)\n\n    param =   {\n        'l2_leaf_reg': trial.suggest_loguniform('l2_leaf_reg', 3.0,5.0),\n        'random_state': trial.suggest_int('random_state', 56,62),       \n        'learning_rate': trial.suggest_loguniform('learning_rate',0.16,0.20),  \n        'bagging_temperature': trial.suggest_loguniform('bagging_temperature', 0.06,0.1),\n        'random_strength':trial.suggest_loguniform('random_strength', 7.0,10.0),    \n        'iterations': trial.suggest_int('iterations', 600,800),  \n\n        #fixed part            \n        'depth': 3,\n        'border_count': 88,         \n        'verbose': False,\n        #'objective': \"binary:logistic\",\n        #'num_class': 1\n    }\n\n    model = catb.CatBoostRegressor(**param) \n    model.fit(train_x,train_y,eval_set=[(test_x,test_y)],early_stopping_rounds=100,verbose=False)\n    preds = model.predict(test_x)\n    rmse = mean_squared_error(test_y, preds,squared=False)\n    \n    return rmse","metadata":{"papermill":{"duration":0.024903,"end_time":"2021-06-30T01:42:35.960585","exception":false,"start_time":"2021-06-30T01:42:35.935682","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-06T01:05:33.969374Z","iopub.execute_input":"2021-10-06T01:05:33.969691Z","iopub.status.idle":"2021-10-06T01:05:33.979265Z","shell.execute_reply.started":"2021-10-06T01:05:33.969657Z","shell.execute_reply":"2021-10-06T01:05:33.978399Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"study = optuna.create_study(direction='minimize')\nstudy.optimize(objective, n_trials=100)\nprint('Number of finished trials:', len(study.trials))\nprint('Best trial:', study.best_trial.params)","metadata":{"papermill":{"duration":1081.471176,"end_time":"2021-06-30T02:00:37.446652","exception":false,"start_time":"2021-06-30T01:42:35.975476","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-06T01:05:33.980663Z","iopub.execute_input":"2021-10-06T01:05:33.981004Z","iopub.status.idle":"2021-10-06T01:06:04.907628Z","shell.execute_reply.started":"2021-10-06T01:05:33.980946Z","shell.execute_reply":"2021-10-06T01:06:04.906705Z"},"trusted":true,"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# shows the scores from all trials\noptuna.visualization.plot_optimization_history(study)","metadata":{"papermill":{"duration":0.167578,"end_time":"2021-06-30T02:00:37.745069","exception":false,"start_time":"2021-06-30T02:00:37.577491","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-06T01:06:04.945048Z","iopub.execute_input":"2021-10-06T01:06:04.945435Z","iopub.status.idle":"2021-10-06T01:06:04.965709Z","shell.execute_reply.started":"2021-10-06T01:06:04.945402Z","shell.execute_reply":"2021-10-06T01:06:04.964726Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# shows the evolution of the search\noptuna.visualization.plot_slice(study)","metadata":{"papermill":{"duration":0.309406,"end_time":"2021-06-30T02:00:38.218357","exception":false,"start_time":"2021-06-30T02:00:37.908951","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-06T01:06:04.994439Z","iopub.execute_input":"2021-10-06T01:06:04.99473Z","iopub.status.idle":"2021-10-06T01:06:05.147511Z","shell.execute_reply.started":"2021-10-06T01:06:04.994699Z","shell.execute_reply":"2021-10-06T01:06:05.146683Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualize parameter importances.\noptuna.visualization.plot_param_importances(study)","metadata":{"papermill":{"duration":0.76143,"end_time":"2021-06-30T02:00:39.149399","exception":false,"start_time":"2021-06-30T02:00:38.387969","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-06T01:06:05.363373Z","iopub.execute_input":"2021-10-06T01:06:05.364008Z","iopub.status.idle":"2021-10-06T01:06:06.148928Z","shell.execute_reply.started":"2021-10-06T01:06:05.363964Z","shell.execute_reply":"2021-10-06T01:06:06.147967Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Best_trial=study.best_trial.params\n\nfix_dict = {\n    'depth': 3,\n    'border_count': 88,         \n    'verbose': False,\n    #'objective': \"binary:logistic\",\n    #'num_class': 1\n  }\n\nBest_trial.update(fix_dict)\n\nprint(Best_trial)","metadata":{"papermill":{"duration":0.04543,"end_time":"2021-06-30T02:00:39.31482","exception":false,"start_time":"2021-06-30T02:00:39.26939","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-06T01:06:06.165997Z","iopub.execute_input":"2021-10-06T01:06:06.166388Z","iopub.status.idle":"2021-10-06T01:06:06.176546Z","shell.execute_reply.started":"2021-10-06T01:06:06.166345Z","shell.execute_reply":"2021-10-06T01:06:06.175546Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample = pd.read_csv(\"/kaggle/input/playground-series-s4e12/sample_submission.csv\")\nprint(sample.shape)","metadata":{"papermill":{"duration":0.178934,"end_time":"2021-06-30T02:00:39.529731","exception":false,"start_time":"2021-06-30T02:00:39.350797","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-06T01:06:06.17769Z","iopub.execute_input":"2021-10-06T01:06:06.178054Z","iopub.status.idle":"2021-10-06T01:06:06.210335Z","shell.execute_reply.started":"2021-10-06T01:06:06.178019Z","shell.execute_reply":"2021-10-06T01:06:06.209301Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preds = np.zeros((sample.shape[0]))\nkf = KFold(n_splits=5,random_state=48,shuffle=True)\nfor trn_idx, test_idx in kf.split(train[columns],target):\n    X_tr,X_val=train[columns].iloc[trn_idx],train[columns].iloc[test_idx]\n    y_tr,y_val=target.iloc[trn_idx],target.iloc[test_idx]\n    model = catb.CatBoostRegressor(**Best_trial)\n    model.fit(X_tr,y_tr,eval_set=[(X_val,y_val)])\n    preds+=model.predict(test[columns])/kf.n_splits   \n    rmse=mean_squared_error(y_val, model.predict(X_val),squared=False)\n    print(rmse)","metadata":{"papermill":{"duration":150.161649,"end_time":"2021-06-30T02:03:09.726111","exception":false,"start_time":"2021-06-30T02:00:39.564462","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-06T01:06:06.243337Z","iopub.execute_input":"2021-10-06T01:06:06.243649Z","iopub.status.idle":"2021-10-06T01:06:15.031383Z","shell.execute_reply.started":"2021-10-06T01:06:06.243617Z","shell.execute_reply":"2021-10-06T01:06:15.030572Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"subm = sample\nsubm['Premium Amount'] = preds.astype(int)\nsubm.to_csv('submission.csv',index=False)\nsubm","metadata":{"papermill":{"duration":1.598507,"end_time":"2021-06-30T02:03:11.447247","exception":false,"start_time":"2021-06-30T02:03:09.84874","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-06T01:06:15.039398Z","iopub.execute_input":"2021-10-06T01:06:15.039732Z","iopub.status.idle":"2021-10-06T01:06:15.165547Z","shell.execute_reply.started":"2021-10-06T01:06:15.039699Z","shell.execute_reply":"2021-10-06T01:06:15.164843Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.037603,"end_time":"2021-06-30T02:03:11.600307","exception":false,"start_time":"2021-06-30T02:03:11.562704","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-10-06T01:06:15.166512Z","iopub.execute_input":"2021-10-06T01:06:15.166896Z","iopub.status.idle":"2021-10-06T01:06:15.17364Z","shell.execute_reply.started":"2021-10-06T01:06:15.166858Z","shell.execute_reply":"2021-10-06T01:06:15.172996Z"},"trusted":true},"outputs":[],"execution_count":null}]}