{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nXGboost stacking\n'''","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.model_selection import train_test_split\nimport os\nfrom tqdm import tqdm\nimport seaborn as sns\nimport xgboost as xgb\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import cohen_kappa_score,classification_report","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/pandaprobatpu1536768models/train_tpu1536_prediction.csv')\nprint(train_df.shape)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(train_df['label'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['label'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df_not_01 = train_df[(train_df['label'] != 0) & (train_df['label'] != 1)]\ntrain_class0 = train_df[train_df['label'] == 0].sample(frac=0.9999)\ntrain_class1 = train_df[train_df['label'] == 1].sample(frac=0.9999)\n\nprint(train_df_not_01['label'].value_counts())\nprint(train_class0['label'].value_counts())\nprint(train_class1['label'].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_train = train_df_not_01\nfinal_train = final_train.append(train_class0)\nfinal_train = final_train.append(train_class1)\nfinal_train.reset_index(inplace = True) \n\nprint(final_train['label'].value_counts())\n#print(final_train.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"xtrain, xval, ytrain, yval = train_test_split(new_train_df[\"image_path\"], new_train_df[[\"col0\",\"col1\",\"col2\",\"col3\",\"col4\"]], test_size = 0.15, stratify = new_train_df[['label']])\n\ndf_train = pd.DataFrame({\"image_path\":xtrain, \"col0\":ytrain['col0'], \"col1\":ytrain['col1'], \"col2\":ytrain['col2'], \"col3\":ytrain['col3'], \"col4\":ytrain['col4'] })\ndf_val = pd.DataFrame({\"image_path\":xval, \"col0\":yval['col0'], \"col1\":yval['col1'], \"col2\":yval['col2'], \"col3\":yval['col3'], \"col4\":yval['col4'] })\n","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"df = pd.read_csv(LABELS).set_index('image_id')\nfiles = sorted(set([p[:32] for p in os.listdir(TRAIN)]))\ndf = df.loc[files]\ndf = df.reset_index()","execution_count":null},{"metadata":{"trusted":true},"cell_type":"markdown","source":"### CV ,  Fold cretion","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\n\ndf = final_train\nnfolds = 10\n\nsplits = StratifiedKFold(n_splits=nfolds, random_state=2020, shuffle=True)\nsplits = list(splits.split(df,df.label))\n\nfolds_splits = np.zeros(len(df)).astype(np.int)\nfor i in range(nfolds): folds_splits[splits[i][1]] = i\n\ndf['split'] = folds_splits\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['split'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_train():\n    for fold_number in range(nfolds):\n        print('Training started for Fold :' + str(fold_number))\n    \n        train_df = df[(df.split != fold_number)]\n        valid_df = df[(df.split == fold_number)]\n        \n        #print(train_df.shape + valid_df.shape )\n    \n        train_model(train_df, valid_df, fold_number)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"    df1 = train_df.iloc[:,1:7]\n    df2 = train_df.iloc[:,7:13]\n    df3 = train_df.iloc[:,13:19]\n    df4 = train_df.iloc[:,19:25]\n    \n        #X_train = (df1.to_numpy() + df2.to_numpy() + df3.to_numpy() + df4.to_numpy()) / 4\n    #y_train = train_df.label.to_numpy()\n    #X_train = train_df.iloc[:,13:19]\n    #y_train = train_df.label\n    \n    xgb_model = xgb.XGBClassifier(n_estimators=100,learning_rate=0.2,objective='multi:softmax',\n                                  max_depth=3, eval_metric = 'mlogloss' )\n    \n    # best score\n    #> 130 4 .3\n    #> 120 4 .3\n    \n    # For 5 fold\n    #({'eta': 0.2, 'max_depth': 4, 'n_estimators': 100}, 0.7147827829075659)\n    # For 20\n    #{'eta': 0.1, 'max_depth': 6, 'n_estimators': 180}0.719783081226504)\n    xgb_model.fit(X_train, y_train, verbose=True)\n\n    vdf1 = valid_df.iloc[:,1:7]\n    vdf2 = valid_df.iloc[:,7:13]\n    vdf3 = valid_df.iloc[:,13:19]\n    vdf4 = valid_df.iloc[:,19:25]\n    \n    X_test = (vdf1.to_numpy() + vdf2.to_numpy() + (1.2*vdf3.to_numpy()) + vdf4.to_numpy()) / 4\n    y_test = valid_df.label.to_numpy()\n    \n    X_test = valid_df.iloc[:,13:19]\n    y_test = valid_df.label\n    predictions = xgb_model.predict(X_test)\n\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = train_df.iloc[:,0:12]\nX_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pickle\ndef train_model(train_df, valid_df, fold_number):\n    \n    X_train = train_df.iloc[:,0:12]\n    y_train = train_df.label\n    \n    xgb_model = xgb.XGBClassifier(n_estimators=180,learning_rate=0.1,objective='multi:softmax',\n                                  max_depth=6, eval_metric = 'mlogloss' )\n  \n\n    xgb_model.fit(X_train, y_train, verbose=True)\n    \n    X_test = valid_df.iloc[:,0:12]\n    y_test = valid_df.label\n    predictions = xgb_model.predict(X_test)\n    \n    #print(predictions)\n    #print(cohen_kappa_score(y_test, predictions,weights='quadratic'))\n    model_score = cohen_kappa_score(y_test, predictions,weights='quadratic')\n    print(model_score)\n    model_name = 'xgboost-' + str(fold_number) + '-' + str(model_score) + '.dat'\n    #pickle.dump(xgb_model, open(model_name, 'wb'))\n    print(classification_report(y_test, predictions))\n    print('---------------------------------------------------------------')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"run_train()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# GridSearch","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV\n\nparams = {\n     \"objective\"    : 'multi:softmax',\n     \"num_classes\"  : 6\n     #\"min_child_weight\" : [ 1, 3, 5, 7 ],\n     #\"gamma\"            : [ 0.0, 0.1, 0.2 , 0.3, 0.4 ],\n     #\"colsample_bytree\" : [ 0.3, 0.4, 0.5 , 0.7 ]\n     }\n\nparameters = {\n     \"eta\"          : [ 0.05, 0.10, 0.15, 0.20, 0.25, 0.30 ] ,\n     \"max_depth\"    : [ 3 , 4, 5, 6],\n     \"n_estimators\" : [50, 60,100,110,120,130,140,150,160,170,180,190,200,210,220]\n     #\"min_child_weight\" : [ 1, 3, 5, 7 ],\n     #\"gamma\"            : [ 0.0, 0.1, 0.2 , 0.3, 0.4 ],\n     #\"colsample_bytree\" : [ 0.3, 0.4, 0.5 , 0.7 ]\n     }\n\nclf = xgb.XGBClassifier(**params)\n\ngrid = GridSearchCV(clf,\n                    parameters, n_jobs=4,\n                    scoring=\"accuracy\",\n                    cv=10)\nX_train = train_df.iloc[:,0:12]\ny_train = train_df.label\n\ngrid.fit(X_train, y_train, verbose=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"import sklearn\nsorted(sklearn.metrics.SCORERS.keys())","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"grid.best_params_, grid.best_score_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"grid.cv_results_","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}