{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30823,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Problem statement\n\nPredict on insurance premiums","metadata":{}},{"cell_type":"markdown","source":"Import libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\n\nfrom scipy.stats import ks_2samp\n\nfrom sklearn.preprocessing import OrdinalEncoder\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\n\nimport statsmodels.api as sm\nimport statsmodels.formula.api as smf\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T20:28:42.629565Z","iopub.execute_input":"2024-12-23T20:28:42.629883Z","iopub.status.idle":"2024-12-23T20:28:43.695786Z","shell.execute_reply.started":"2024-12-23T20:28:42.629859Z","shell.execute_reply":"2024-12-23T20:28:43.695051Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Use os to retrieve files used in competition","metadata":{}},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T20:28:43.696697Z","iopub.execute_input":"2024-12-23T20:28:43.696997Z","iopub.status.idle":"2024-12-23T20:28:43.702781Z","shell.execute_reply.started":"2024-12-23T20:28:43.696977Z","shell.execute_reply":"2024-12-23T20:28:43.701972Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Use pandas to read files","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")\nsubmission = pd.read_csv(\"/kaggle/input/playground-series-s4e12/sample_submission.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T20:28:43.704228Z","iopub.execute_input":"2024-12-23T20:28:43.704473Z","iopub.status.idle":"2024-12-23T20:28:49.669017Z","shell.execute_reply.started":"2024-12-23T20:28:43.704444Z","shell.execute_reply":"2024-12-23T20:28:49.668012Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"View all columns","metadata":{}},{"cell_type":"code","source":"pd.set_option('display.max_columns', None)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T20:28:49.670453Z","iopub.execute_input":"2024-12-23T20:28:49.670758Z","iopub.status.idle":"2024-12-23T20:28:49.674445Z","shell.execute_reply.started":"2024-12-23T20:28:49.670736Z","shell.execute_reply":"2024-12-23T20:28:49.673522Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T20:28:49.675514Z","iopub.execute_input":"2024-12-23T20:28:49.675839Z","iopub.status.idle":"2024-12-23T20:28:49.709611Z","shell.execute_reply.started":"2024-12-23T20:28:49.6758Z","shell.execute_reply":"2024-12-23T20:28:49.708711Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T20:28:49.710342Z","iopub.execute_input":"2024-12-23T20:28:49.71066Z","iopub.status.idle":"2024-12-23T20:28:50.240556Z","shell.execute_reply.started":"2024-12-23T20:28:49.710628Z","shell.execute_reply":"2024-12-23T20:28:50.239709Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T20:28:50.241407Z","iopub.execute_input":"2024-12-23T20:28:50.241747Z","iopub.status.idle":"2024-12-23T20:28:50.76604Z","shell.execute_reply.started":"2024-12-23T20:28:50.241714Z","shell.execute_reply":"2024-12-23T20:28:50.765141Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T20:28:50.766848Z","iopub.execute_input":"2024-12-23T20:28:50.767109Z","iopub.status.idle":"2024-12-23T20:28:50.790967Z","shell.execute_reply.started":"2024-12-23T20:28:50.767087Z","shell.execute_reply":"2024-12-23T20:28:50.790261Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T20:28:50.79345Z","iopub.execute_input":"2024-12-23T20:28:50.793709Z","iopub.status.idle":"2024-12-23T20:28:50.82196Z","shell.execute_reply.started":"2024-12-23T20:28:50.793689Z","shell.execute_reply":"2024-12-23T20:28:50.82118Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T20:28:50.823669Z","iopub.execute_input":"2024-12-23T20:28:50.823975Z","iopub.status.idle":"2024-12-23T20:28:51.181008Z","shell.execute_reply.started":"2024-12-23T20:28:50.823954Z","shell.execute_reply":"2024-12-23T20:28:51.180109Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T20:28:51.181767Z","iopub.execute_input":"2024-12-23T20:28:51.181981Z","iopub.status.idle":"2024-12-23T20:28:51.191095Z","shell.execute_reply.started":"2024-12-23T20:28:51.181963Z","shell.execute_reply":"2024-12-23T20:28:51.190369Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Impute missing values in train and test set","metadata":{}},{"cell_type":"code","source":"for col in train:\n    if train[col].dtype == 'object':\n        train[col] = train[col].fillna('Not recorded')\n    if train[col].dtype == 'float':\n        train[col] = train[col].fillna(999)\n    if train[col].dtype == 'int':\n        train[col] = train[col].fillna(999)\n        \nfor col in test:\n    if test[col].dtype == 'object':\n        test[col] = test[col].fillna('Not recorded')\n    if test[col].dtype == 'float':\n        test[col] = test[col].fillna(999)\n    if test[col].dtype == 'int':\n        test[col] = test[col].fillna(999)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T20:28:51.191973Z","iopub.execute_input":"2024-12-23T20:28:51.192252Z","iopub.status.idle":"2024-12-23T20:28:52.55782Z","shell.execute_reply.started":"2024-12-23T20:28:51.19222Z","shell.execute_reply":"2024-12-23T20:28:52.55686Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.isna().sum().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T20:28:52.558784Z","iopub.execute_input":"2024-12-23T20:28:52.559109Z","iopub.status.idle":"2024-12-23T20:28:53.088839Z","shell.execute_reply.started":"2024-12-23T20:28:52.55908Z","shell.execute_reply":"2024-12-23T20:28:53.087925Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.isna().sum().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T20:28:53.089836Z","iopub.execute_input":"2024-12-23T20:28:53.090148Z","iopub.status.idle":"2024-12-23T20:28:53.443238Z","shell.execute_reply.started":"2024-12-23T20:28:53.090124Z","shell.execute_reply":"2024-12-23T20:28:53.442332Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Compare train and test datasets to see if they come from the same distribution","metadata":{}},{"cell_type":"code","source":"to_drop = []\n\nfor col in test:\n    stat, pv = ks_2samp(train[col], test[col])\n    if pv < 0.05:\n        to_drop.append(col)\nprint(to_drop)\n\ntrain.drop(to_drop,axis=1,inplace=True)\ntest.drop(to_drop,axis=1,inplace=True)\n\nprint(train.shape, test.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T20:28:53.44401Z","iopub.execute_input":"2024-12-23T20:28:53.444234Z","iopub.status.idle":"2024-12-23T20:29:53.17493Z","shell.execute_reply.started":"2024-12-23T20:28:53.444215Z","shell.execute_reply":"2024-12-23T20:29:53.173791Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Ordinal encode train and test sets","metadata":{}},{"cell_type":"code","source":"enc = OrdinalEncoder(handle_unknown='use_encoded_value', unknown_value=-1)\n\nfor col in train:\n    if train[col].dtype == 'object':\n        train[col] = enc.fit_transform(train[col].values.reshape(-1,1))\n        test[col] = enc.transform(test[col].values.reshape(-1,1))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T20:29:53.175966Z","iopub.execute_input":"2024-12-23T20:29:53.176223Z","iopub.status.idle":"2024-12-23T20:29:58.722517Z","shell.execute_reply.started":"2024-12-23T20:29:53.176202Z","shell.execute_reply":"2024-12-23T20:29:58.721777Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Analyse target","metadata":{}},{"cell_type":"code","source":"plt.hist(train['Premium Amount'])\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T20:29:58.723191Z","iopub.execute_input":"2024-12-23T20:29:58.723425Z","iopub.status.idle":"2024-12-23T20:29:58.887329Z","shell.execute_reply.started":"2024-12-23T20:29:58.723405Z","shell.execute_reply":"2024-12-23T20:29:58.886365Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Heatmap","metadata":{}},{"cell_type":"code","source":"corr = train.corr()\nsns.heatmap(corr)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T20:29:58.888081Z","iopub.execute_input":"2024-12-23T20:29:58.888376Z","iopub.status.idle":"2024-12-23T20:30:00.516546Z","shell.execute_reply.started":"2024-12-23T20:29:58.888353Z","shell.execute_reply":"2024-12-23T20:30:00.515671Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Define X and y variables","metadata":{}},{"cell_type":"code","source":"y = train.pop('Premium Amount')\nX = np.array(train)\nX_test = np.array(test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T20:30:00.51738Z","iopub.execute_input":"2024-12-23T20:30:00.517629Z","iopub.status.idle":"2024-12-23T20:30:00.721993Z","shell.execute_reply.started":"2024-12-23T20:30:00.517585Z","shell.execute_reply":"2024-12-23T20:30:00.721256Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Split X into training and validation sets","metadata":{}},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split( X, y, test_size=0.1, shuffle=True, random_state=42)\nX_train.shape, y_train.shape, X_val.shape, y_val.shape, X_test.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T20:30:38.920826Z","iopub.execute_input":"2024-12-23T20:30:38.921126Z","iopub.status.idle":"2024-12-23T20:30:39.268572Z","shell.execute_reply.started":"2024-12-23T20:30:38.921105Z","shell.execute_reply":"2024-12-23T20:30:39.267459Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Define model","metadata":{}},{"cell_type":"code","source":"#model\nmodel = sm.OLS(y_train, X_train).fit()\nprint(model.summary())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T20:30:49.863815Z","iopub.execute_input":"2024-12-23T20:30:49.864131Z","iopub.status.idle":"2024-12-23T20:30:52.992573Z","shell.execute_reply.started":"2024-12-23T20:30:49.864107Z","shell.execute_reply":"2024-12-23T20:30:52.989834Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#residuals greater than 3 is said to be an outlier\n#create instance of influence\ninfluence = model.get_influence()\n\n#obtain standardized residuals\nstandardized_residuals = influence.resid_studentized_internal\n\n#display standardized residuals\nprint(standardized_residuals)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T20:31:03.672831Z","iopub.execute_input":"2024-12-23T20:31:03.673133Z","iopub.status.idle":"2024-12-23T20:31:03.860078Z","shell.execute_reply.started":"2024-12-23T20:31:03.673112Z","shell.execute_reply":"2024-12-23T20:31:03.859075Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sm.qqplot(standardized_residuals,line='45')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T20:31:13.400505Z","iopub.execute_input":"2024-12-23T20:31:13.400867Z","iopub.status.idle":"2024-12-23T20:31:19.491825Z","shell.execute_reply.started":"2024-12-23T20:31:13.400839Z","shell.execute_reply":"2024-12-23T20:31:19.49081Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred = model.predict(X_val)\ny_pred","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T20:31:25.713192Z","iopub.execute_input":"2024-12-23T20:31:25.713492Z","iopub.status.idle":"2024-12-23T20:31:25.724466Z","shell.execute_reply.started":"2024-12-23T20:31:25.71347Z","shell.execute_reply":"2024-12-23T20:31:25.723291Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mse = mean_squared_error(y_val, y_pred)\nrmse = np.sqrt(mse)\nrmsle = np.log(rmse)\nrmsle","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T20:32:55.753714Z","iopub.execute_input":"2024-12-23T20:32:55.754053Z","iopub.status.idle":"2024-12-23T20:32:55.761001Z","shell.execute_reply.started":"2024-12-23T20:32:55.754026Z","shell.execute_reply":"2024-12-23T20:32:55.760132Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, ax = plt.subplots()\nax.scatter(y_val, y_pred, edgecolors=(0, 0, 0))\nax.plot([y.min(), y.max()], [y.min(), y.max()], 'k--', lw=4)\nax.set_xlabel('Measured')\nax.set_ylabel('Predicted')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T20:33:03.99112Z","iopub.execute_input":"2024-12-23T20:33:03.991414Z","iopub.status.idle":"2024-12-23T20:33:04.353375Z","shell.execute_reply.started":"2024-12-23T20:33:03.991392Z","shell.execute_reply":"2024-12-23T20:33:04.352421Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Predict on X_test","metadata":{}},{"cell_type":"code","source":"pred = model.predict(X_test)\npred","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T20:33:31.262974Z","iopub.execute_input":"2024-12-23T20:33:31.263276Z","iopub.status.idle":"2024-12-23T20:33:31.278733Z","shell.execute_reply.started":"2024-12-23T20:33:31.263253Z","shell.execute_reply":"2024-12-23T20:33:31.27742Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Prepare submission","metadata":{}},{"cell_type":"code","source":"submission['Premium Amount'] = pred\nsubmission.to_csv('submission.csv', index=False)\nsubmission = pd.read_csv('submission.csv')\nsubmission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T20:33:36.422752Z","iopub.execute_input":"2024-12-23T20:33:36.423059Z","iopub.status.idle":"2024-12-23T20:33:38.000472Z","shell.execute_reply.started":"2024-12-23T20:33:36.423039Z","shell.execute_reply":"2024-12-23T20:33:37.999641Z"}},"outputs":[],"execution_count":null}]}