{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Linear Regression\n\nExperiments with both mean imputation and pairwise deletion","metadata":{}},{"cell_type":"markdown","source":"## Feature Engineering\n\nOne-hot encoding","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\ndf_train = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\")\ndf_test = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:54:26.762304Z","iopub.execute_input":"2024-12-19T01:54:26.762687Z","iopub.status.idle":"2024-12-19T01:54:36.661236Z","shell.execute_reply.started":"2024-12-19T01:54:26.762657Z","shell.execute_reply":"2024-12-19T01:54:36.660414Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import OneHotEncoder\n\ncat = [\"Gender\", \"Marital Status\", \"Education Level\", \"Occupation\", \"Location\", \"Policy Type\", \n       \"Customer Feedback\", \"Smoking Status\", \"Exercise Frequency\", \"Property Type\"]\n\nohe = OneHotEncoder()\nresult_train = ohe.fit_transform(df_train[cat])\nresult_test = ohe.transform(df_test[cat])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:54:37.661025Z","iopub.execute_input":"2024-12-19T01:54:37.661395Z","iopub.status.idle":"2024-12-19T01:54:42.8379Z","shell.execute_reply.started":"2024-12-19T01:54:37.661365Z","shell.execute_reply":"2024-12-19T01:54:42.836805Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ohe.categories_","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:54:45.158303Z","iopub.execute_input":"2024-12-19T01:54:45.15883Z","iopub.status.idle":"2024-12-19T01:54:45.16612Z","shell.execute_reply.started":"2024-12-19T01:54:45.158791Z","shell.execute_reply":"2024-12-19T01:54:45.165047Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i,feature_name in enumerate(ohe.get_feature_names_out()):\n    df_train[feature_name] = result_train[:,i].toarray()\n    df_test[feature_name] = result_test[:,i].toarray()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:54:49.7565Z","iopub.execute_input":"2024-12-19T01:54:49.756841Z","iopub.status.idle":"2024-12-19T01:54:52.868103Z","shell.execute_reply.started":"2024-12-19T01:54:49.756815Z","shell.execute_reply":"2024-12-19T01:54:52.867161Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from datetime import datetime\n# Encode time\n\nepoch_time_train = [(datetime.strptime(i, \"%Y-%m-%d %H:%M:%S.%f\") - datetime(1970, 1, 1)).total_seconds() for i in df_train[\"Policy Start Date\"]]\nepoch_time_test  = [(datetime.strptime(i, \"%Y-%m-%d %H:%M:%S.%f\") - datetime(1970, 1, 1)).total_seconds() for i in df_test[\"Policy Start Date\"]]\ndf_train[\"Policy Start Date Epoch\"] = epoch_time_train\ndf_test[\"Policy Start Date Epoch\"] = epoch_time_test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:54:55.355812Z","iopub.execute_input":"2024-12-19T01:54:55.356165Z","iopub.status.idle":"2024-12-19T01:55:16.01491Z","shell.execute_reply.started":"2024-12-19T01:54:55.35614Z","shell.execute_reply":"2024-12-19T01:55:16.013976Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Mean Imputation","metadata":{}},{"cell_type":"code","source":"import statsmodels.api as sm\nfrom statsmodels.formula.api import ols\n\nimport numpy as np\n\ndf_train_mean_impute = df_train.copy()\n\n# Mean Imputation\ncont_keys = [\"Age\", \"Annual Income\", \"Number of Dependents\", \"Health Score\", \"Previous Claims\", \n         \"Vehicle Age\", \"Credit Score\", \"Insurance Duration\"]\nfor key in cont_keys:\n    df_train_mean_impute[key] = df_train_mean_impute[key].fillna(df_train_mean_impute[key].mean())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:55:19.643884Z","iopub.execute_input":"2024-12-19T01:55:19.644243Z","iopub.status.idle":"2024-12-19T01:55:21.332818Z","shell.execute_reply.started":"2024-12-19T01:55:19.644217Z","shell.execute_reply":"2024-12-19T01:55:21.331832Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train_mean_impute[\"Nat. Log. Premium Amount\"] = np.log(df_train_mean_impute[\"Premium Amount\"])\ndf_train_mean_impute[\"Nat. Log. Income\"] = np.log(df_train_mean_impute[\"Annual Income\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:55:27.088025Z","iopub.execute_input":"2024-12-19T01:55:27.088541Z","iopub.status.idle":"2024-12-19T01:55:27.123062Z","shell.execute_reply.started":"2024-12-19T01:55:27.088513Z","shell.execute_reply":"2024-12-19T01:55:27.1222Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Careful not to include all columns for each one-hot encoded category\nkeysX = [\"Age\", \"Annual Income\", \"Nat. Log. Income\", \"Number of Dependents\", \"Health Score\", \"Previous Claims\", \n         \"Vehicle Age\", \"Credit Score\", \"Insurance Duration\", \"Gender_Male\", \n         \"Marital Status_Married\", \"Marital Status_Single\", \"Marital Status_nan\",\n         \"Education Level_High School\", \"Education Level_Master\\'s\", \"Education Level_PhD\", \n         \"Occupation_Self-Employed\", \"Occupation_Unemployed\", \"Occupation_nan\", \n         \"Location_Suburban\", \"Location_Urban\", \"Policy Type_Comprehensive\", \n         \"Policy Type_Premium\", \"Customer Feedback_Good\", \"Customer Feedback_Poor\",\n         \"Customer Feedback_nan\", \"Smoking Status_Yes\", \"Exercise Frequency_Monthly\", \n         \"Exercise Frequency_Rarely\", \"Exercise Frequency_Weekly\", \"Property Type_Condo\",\n         \"Property Type_House\"]\nkeyY = \"Nat. Log. Premium Amount\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:55:33.732844Z","iopub.execute_input":"2024-12-19T01:55:33.733198Z","iopub.status.idle":"2024-12-19T01:55:33.73832Z","shell.execute_reply.started":"2024-12-19T01:55:33.733171Z","shell.execute_reply":"2024-12-19T01:55:33.737284Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Y = df_train_mean_impute[keyY]\nX = df_train_mean_impute[keysX]\nX = sm.add_constant(X)\n\nmodel = sm.OLS(Y, X)\nresults = model.fit()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:55:40.045403Z","iopub.execute_input":"2024-12-19T01:55:40.045738Z","iopub.status.idle":"2024-12-19T01:55:43.818616Z","shell.execute_reply.started":"2024-12-19T01:55:40.045715Z","shell.execute_reply":"2024-12-19T01:55:43.817324Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"results.summary()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:55:43.820066Z","iopub.execute_input":"2024-12-19T01:55:43.820495Z","iopub.status.idle":"2024-12-19T01:55:44.100742Z","shell.execute_reply.started":"2024-12-19T01:55:43.820457Z","shell.execute_reply":"2024-12-19T01:55:44.098718Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from io import StringIO\ndf_results = pd.read_csv(StringIO(results.summary().tables[1].as_csv()))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:55:54.563912Z","iopub.execute_input":"2024-12-19T01:55:54.564247Z","iopub.status.idle":"2024-12-19T01:55:54.657178Z","shell.execute_reply.started":"2024-12-19T01:55:54.564221Z","shell.execute_reply":"2024-12-19T01:55:54.656065Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_results.iloc[[11,12,13, 17, 18, 19, 24, 25, 26]]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:56:00.585603Z","iopub.execute_input":"2024-12-19T01:56:00.585983Z","iopub.status.idle":"2024-12-19T01:56:00.608166Z","shell.execute_reply.started":"2024-12-19T01:56:00.585955Z","shell.execute_reply":"2024-12-19T01:56:00.607171Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Pairwise Deletion\n\nSome issues here: it is unclear how to obtain standard error of coefficients, so use nested models to compare inclusion vs exclusion of columns","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\ndf_train_pairwise_delete = df_train.copy()\n\ndf_train_pairwise_delete[\"Nat. Log. Premium Amount\"] = np.log(df_train_pairwise_delete[\"Premium Amount\"])\ndf_train_pairwise_delete[\"Nat. Log. Income\"] = np.log(df_train_pairwise_delete[\"Annual Income\"])\ndf_train_pairwise_delete[\"Constant\"] = 1","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:56:43.705317Z","iopub.execute_input":"2024-12-19T01:56:43.705686Z","iopub.status.idle":"2024-12-19T01:56:44.373796Z","shell.execute_reply.started":"2024-12-19T01:56:43.705654Z","shell.execute_reply":"2024-12-19T01:56:44.372664Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"keysX = [\"Age\", \"Annual Income\", \"Number of Dependents\", \"Health Score\", \"Previous Claims\", \n         \"Vehicle Age\", \"Credit Score\", \"Insurance Duration\", \"Gender_Male\", \n         \"Marital Status_Married\", \"Marital Status_Single\", \"Marital Status_nan\",\n         \"Education Level_High School\", \"Education Level_Master\\'s\", \"Education Level_PhD\", \n         \"Occupation_Self-Employed\", \"Occupation_Unemployed\", \"Occupation_nan\", \n         \"Location_Suburban\", \"Location_Urban\", \"Policy Type_Comprehensive\", \n         \"Policy Type_Premium\", \"Customer Feedback_Good\", \"Customer Feedback_Poor\",\n         \"Customer Feedback_nan\", \"Smoking Status_Yes\", \"Exercise Frequency_Monthly\", \n         \"Exercise Frequency_Rarely\", \"Exercise Frequency_Weekly\", \"Property Type_Condo\",\n         \"Property Type_House\", \"Policy Start Date Epoch\", \"Nat. Log. Income\", \"Constant\"]\nkeyY = \"Nat. Log. Premium Amount\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:56:50.136746Z","iopub.execute_input":"2024-12-19T01:56:50.137145Z","iopub.status.idle":"2024-12-19T01:56:50.142252Z","shell.execute_reply.started":"2024-12-19T01:56:50.13712Z","shell.execute_reply":"2024-12-19T01:56:50.141284Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\n\n# Returns [Coefficients, SSREG, SSERR, SSTOT]\ndef pairwise_deletion(df: pd.DataFrame, keysX: list[str], keyY: list[str], impute_mean_eval: bool=False) -> tuple[np.ndarray, float, float, float]:\n    Sxx = np.zeros((len(keysX), len(keysX)))\n    Sxy = np.zeros((len(keysX), ))\n    for i,keyi in enumerate(keysX):\n        Sxy[i] = np.mean(df[keyi]*df[keyY])\n        for j,keyj in enumerate(keysX):\n            Sxx[i][j] = np.mean(df[keyi]*df[keyj])\n\n    df_ss = df[keysX+[keyY]]\n    if impute_mean_eval:\n        for k in keysX:\n            df_ss.loc[:,k] = df_ss[k].fillna(np.mean(df_ss[k]))\n    else:\n        df_ss = df_ss.dropna()\n    \n    beta = np.matmul(np.linalg.inv(Sxx), Sxy)\n    SSreg = np.sum((np.sum(beta * np.array(df_ss[keysX]), axis=1) - np.mean(df_ss[keyY]))**2)\n    SStot = np.sum((df_ss[keyY] - np.mean(df_ss[keyY]))**2)\n    SSerr = SStot - SSreg\n\n    return beta, SSreg, SSerr, SStot\n\npairwise_deletion(df_train_pairwise_delete, keysX, keyY, True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:56:59.827386Z","iopub.execute_input":"2024-12-19T01:56:59.82791Z","iopub.status.idle":"2024-12-19T01:57:07.933168Z","shell.execute_reply.started":"2024-12-19T01:56:59.827862Z","shell.execute_reply":"2024-12-19T01:57:07.93198Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"exclude = [[\"Age\"], [\"Annual Income\"], [\"Number of Dependents\"], [\"Health Score\"], [\"Previous Claims\"], \n         [\"Vehicle Age\"], [\"Credit Score\"], [\"Insurance Duration\"], [\"Nat. Log. Income\"], [\"Policy Start Date Epoch\"],\n         [\"Gender_Male\"], \n         [\"Marital Status_Married\", \"Marital Status_Single\", \"Marital Status_nan\"],\n         [\"Education Level_High School\", \"Education Level_Master\\'s\", \"Education Level_PhD\"], \n         [\"Occupation_Self-Employed\", \"Occupation_Unemployed\", \"Occupation_nan\"], \n         [\"Location_Suburban\", \"Location_Urban\"], \n         [\"Policy Type_Comprehensive\", \"Policy Type_Premium\"],\n         [\"Customer Feedback_Good\", \"Customer Feedback_Poor\", \"Customer Feedback_nan\"], \n         [\"Smoking Status_Yes\"], \n         [\"Exercise Frequency_Monthly\", \"Exercise Frequency_Rarely\", \"Exercise Frequency_Weekly\"], \n         [\"Property Type_Condo\", \"Property Type_House\"],\n         [\"Marital Status_nan\"],\n         [\"Customer Feedback_nan\"],\n         [\"Occupation_nan\"]]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:57:10.937198Z","iopub.execute_input":"2024-12-19T01:57:10.937623Z","iopub.status.idle":"2024-12-19T01:57:10.944004Z","shell.execute_reply.started":"2024-12-19T01:57:10.937594Z","shell.execute_reply":"2024-12-19T01:57:10.942553Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Testing significance of each subset of features.","metadata":{}},{"cell_type":"code","source":"full_beta, full_ss_reg, full_ss_err, full_ss_tot = pairwise_deletion(df_train_pairwise_delete, keysX, keyY, True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:57:21.672697Z","iopub.execute_input":"2024-12-19T01:57:21.6731Z","iopub.status.idle":"2024-12-19T01:57:30.626479Z","shell.execute_reply.started":"2024-12-19T01:57:21.673071Z","shell.execute_reply":"2024-12-19T01:57:30.6254Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for e in exclude:\n    nested = list(set(keysX) - set(e))\n    nested_beta, nested_ss_reg, nested_ss_err, nesteed_ss_tot = pairwise_deletion(df_train_pairwise_delete, nested, keyY, True)\n    ss_ex = full_ss_reg - nested_ss_reg\n    \n    F = max(0, (ss_ex/(len(keysX) - len(nested))) / (full_ss_err / (len(df_train_pairwise_delete) - len(keysX) - 1)))\n    print(F, e)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T01:59:16.767997Z","iopub.execute_input":"2024-12-19T01:59:16.76834Z","iopub.status.idle":"2024-12-19T02:02:14.579912Z","shell.execute_reply.started":"2024-12-19T01:59:16.768314Z","shell.execute_reply":"2024-12-19T02:02:14.578709Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# R^2\nfull_ss_reg/full_ss_tot","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-19T02:02:14.581463Z","iopub.execute_input":"2024-12-19T02:02:14.581888Z","iopub.status.idle":"2024-12-19T02:02:14.588201Z","shell.execute_reply.started":"2024-12-19T02:02:14.581846Z","shell.execute_reply":"2024-12-19T02:02:14.587097Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}