{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\nimport numpy as np \nimport pandas as pd \nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nimport seaborn as sns\nimport os\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score, explained_variance_score\nfrom sklearn.metrics import mean_absolute_percentage_error\nfrom sklearn.model_selection import train_test_split\nfrom catboost import CatBoostRegressor\nimport lightgbm as lgb\nimport xgboost as xgb\nimport matplotlib.pyplot as plt\nimport warnings\n\nwarnings.filterwarnings('ignore', category=FutureWarning) \ntrain = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')\ntest = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-30T20:35:13.74469Z","iopub.execute_input":"2024-12-30T20:35:13.745015Z","iopub.status.idle":"2024-12-30T20:35:20.859515Z","shell.execute_reply.started":"2024-12-30T20:35:13.74499Z","shell.execute_reply":"2024-12-30T20:35:20.858453Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.set_option('display.max_columns', None)\nprint(train.info())\nprint(test.info())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T20:35:20.86084Z","iopub.execute_input":"2024-12-30T20:35:20.861216Z","iopub.status.idle":"2024-12-30T20:35:21.941993Z","shell.execute_reply.started":"2024-12-30T20:35:20.861189Z","shell.execute_reply":"2024-12-30T20:35:21.941162Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\nplt.figure(figsize=(20, 12)) \nsubplot_num = 1\n\nfor i, col in enumerate(train.columns):\n   if train[col].isnull().any():\n       plt.subplot(3, 4, subplot_num)  \n       sns.histplot(data=train, x=col, bins=30)\n          \n       subplot_num +=1\n       \nplt.subplot(3, 4, subplot_num)  \nsns.histplot(data=train, x='Premium Amount', bins=30)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T20:35:21.943717Z","iopub.execute_input":"2024-12-30T20:35:21.943981Z","iopub.status.idle":"2024-12-30T20:35:31.779207Z","shell.execute_reply.started":"2024-12-30T20:35:21.943957Z","shell.execute_reply":"2024-12-30T20:35:31.778119Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def impute_missing_data(df):\n    objects = df.select_dtypes(include=['object']).columns\n    for obj in objects:\n        df[obj]= df[obj].fillna('Unknown')\n    \n    numerical = df.select_dtypes(exclude=['object']).columns\n    for number in numerical:\n        df[number] = df[number].fillna(-1)\n\n    return df\ntrain = impute_missing_data(train)\ntest = impute_missing_data(test)\n\n# log transform skewed distributions\ntrain['Annual Income'] = train['Annual Income'].apply(lambda x: np.log1p(x) if x > -1 else -1) \ntest['Annual Income'] = test['Annual Income'].apply(lambda x: np.log1p(x) if x > -1 else -1)\n\ntrain['Premium Amount'] = np.log1p(train['Premium Amount'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T20:35:31.780825Z","iopub.execute_input":"2024-12-30T20:35:31.781154Z","iopub.status.idle":"2024-12-30T20:35:36.108771Z","shell.execute_reply.started":"2024-12-30T20:35:31.781116Z","shell.execute_reply":"2024-12-30T20:35:36.1079Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T20:35:36.109706Z","iopub.execute_input":"2024-12-30T20:35:36.11Z","iopub.status.idle":"2024-12-30T20:35:36.141291Z","shell.execute_reply.started":"2024-12-30T20:35:36.109976Z","shell.execute_reply":"2024-12-30T20:35:36.140177Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create better formatting for date\ndef process_dates(df):\n    df['Policy Start Date'] = pd.to_datetime(df['Policy Start Date'])\n    df['Year'] = df['Policy Start Date'].dt.year\n    df['Month'] = df['Policy Start Date'].dt.month\n    df['Day'] = df['Policy Start Date'].dt.day\n    \n  \n\n    df.drop(['Policy Start Date'], axis=1, inplace=True)\n    return df\n    \ntrain = process_dates(train)\ntest = process_dates(test)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T20:35:36.142462Z","iopub.execute_input":"2024-12-30T20:35:36.142806Z","iopub.status.idle":"2024-12-30T20:35:37.550305Z","shell.execute_reply.started":"2024-12-30T20:35:36.14274Z","shell.execute_reply":"2024-12-30T20:35:37.549031Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Scale the numerical values\n\nscaler = StandardScaler()\npremium_amount = train['Premium Amount'].copy()\ntrain.drop(['Premium Amount','id'], axis=1, inplace=True)\ntest_id = test['id'].copy()\ntest.drop(['id'], axis=1, inplace=True)\n\nnumerical_columns = train.select_dtypes(exclude=['object', 'datetime', 'bool']).columns.tolist()\n\ntrain[numerical_columns]= scaler.fit_transform(train[numerical_columns])\ntest[numerical_columns] = scaler.transform(test[numerical_columns])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T20:35:37.550968Z","iopub.execute_input":"2024-12-30T20:35:37.551249Z","iopub.status.idle":"2024-12-30T20:35:38.258034Z","shell.execute_reply.started":"2024-12-30T20:35:37.551225Z","shell.execute_reply":"2024-12-30T20:35:38.256924Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train.select_dtypes(include=['object']).columns)\n\n# Encode ordinal categories\ndef encode_ordinal(df):\n    policy_map = {'Unknown': 0, 'Basic': 0, 'Comprehensive': 1, 'Premium': 2}\n    exercise_map = {'Unknown': 0, 'Rarely': 0, 'Monthly': 1, 'Weekly':2, 'Daily': 3}\n    feedback_map = {'Unknown': 0, 'Poor': 0, 'Average': 1, 'Good': 2}\n    education_map = {'Unknown': 0, 'High School': 0, 'Bachelor\\'s': 1, 'Master\\'s': 2, 'PhD': 3 }\n\n    df['Policy Type'] = df['Policy Type'].map(policy_map)\n    df['Exercise Frequency'] = df['Exercise Frequency'].map(exercise_map)\n    df['Customer Feedback'] = df['Customer Feedback'].map(feedback_map)\n    df['Education Level'] = df['Education Level'].map(education_map)\n    \n    \n    return df\n\ntrain = encode_ordinal(train)\ntest = encode_ordinal(test)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T20:35:38.2612Z","iopub.execute_input":"2024-12-30T20:35:38.261467Z","iopub.status.idle":"2024-12-30T20:35:39.145571Z","shell.execute_reply.started":"2024-12-30T20:35:38.261444Z","shell.execute_reply":"2024-12-30T20:35:39.144696Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Encode nominal categories\n\nnominal = train.select_dtypes(include=['object','category']).columns.difference(['Policy Type', 'Exercise Frequency', 'Customer Feedback', 'Education Level'])\nprint(nominal)\ndef encode_nominal(df, nominal):\n    for col in nominal:\n        df = pd.concat([df, pd.get_dummies(df[col], prefix=col)], axis=1)\n        df.drop(col, axis=1, inplace=True)\n\n    return df\n    \ntrain = encode_nominal(train, nominal)\ntest = encode_nominal(test, nominal)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T20:35:39.147067Z","iopub.execute_input":"2024-12-30T20:35:39.147334Z","iopub.status.idle":"2024-12-30T20:35:42.042126Z","shell.execute_reply.started":"2024-12-30T20:35:39.147311Z","shell.execute_reply":"2024-12-30T20:35:42.041241Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create features \n\ndef features(df):\n    df['Annual Income per Dependent'] = df['Annual Income'] / (df['Number of Dependents'] +1)\n    df['Health Overall'] = (df['Health Score'] * df['Exercise Frequency']*.8) - (df['Smoking Status_Yes'] *.2)\n    df['Age Group'] = pd.qcut(df['Age'], q=5, \n                             labels=['Very Young', 'Young', 'Middle', 'Senior', 'Elderly'])\n    df['Age and Health'] = df['Age'] * df['Health Score']\n    df['Income to Age'] = df['Annual Income'] / (df['Age'] +1)\n    df['Vehicle Income Ratio'] = df['Vehicle Age'] / (df['Annual Income'] + 1)\n    df['Fiancial Stability'] = df['Credit Score']*.4 + df['Annual Income']*.4 + df['Insurance Duration'] * .2\n    df['Property Risk'] = df['Previous Claims'] * 0.4 + df['Credit Score'] * 0.3 +df['Insurance Duration'] * 0.3\n    df['Claim per Year'] = df['Previous Claims'] / (df['Insurance Duration'] +1)\n    \n    return df\ntrain = features(train)\ntest = features(test)\ntrain.drop(['Exercise Frequency'], axis=1,inplace=True)\ntest.drop(['Exercise Frequency'], axis=1,inplace=True)\n\nprint(train.info())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T20:35:42.043131Z","iopub.execute_input":"2024-12-30T20:35:42.04338Z","iopub.status.idle":"2024-12-30T20:35:42.501612Z","shell.execute_reply.started":"2024-12-30T20:35:42.043358Z","shell.execute_reply":"2024-12-30T20:35:42.500455Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T20:35:42.50261Z","iopub.execute_input":"2024-12-30T20:35:42.502977Z","iopub.status.idle":"2024-12-30T20:35:42.543585Z","shell.execute_reply.started":"2024-12-30T20:35:42.502941Z","shell.execute_reply":"2024-12-30T20:35:42.542561Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = train\ny = premium_amount\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T20:35:42.544509Z","iopub.execute_input":"2024-12-30T20:35:42.54478Z","iopub.status.idle":"2024-12-30T20:35:43.021476Z","shell.execute_reply.started":"2024-12-30T20:35:42.544756Z","shell.execute_reply":"2024-12-30T20:35:43.020384Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Testing Models\n\"\"\"\n\nmodels = {\n    'lightgbm': lgb.LGBMRegressor(\n        n_estimators=1000,\n        learning_rate=0.05,\n        num_leaves=31,\n        subsample=0.8,\n        colsample_bytree=0.8,\n        random_state=42\n    ),\n    'xgboost': xgb.XGBRegressor(\n        n_estimators=1000,\n        learning_rate=0.05,\n        max_depth=6,\n        subsample=0.8,\n        colsample_bytree=0.8,\n        tree_method='hist',  # More memory efficient\n        random_state=42\n    ),\n    'catboost': CatBoostRegressor(\n        iterations=1000,\n        learning_rate=0.05,\n        depth=6,\n        subsample=0.8,\n        verbose=False\n    )\n}\n\n\nfor model_name, model in models.items():\n    model.fit(X_train, y_train)\n    \n    y_pred = model.predict(X_test) \n    \n\n    rmsle = np.sqrt(np.mean((y_pred - y_test) ** 2))\n    mae = mean_absolute_error(y_test, y_pred)\n    mse = mean_squared_error(y_test, y_pred)\n    r2 = r2_score(y_test, y_pred)\n    explained_variance = explained_variance_score(y_test, y_pred)\n    mape = mean_absolute_percentage_error(y_test, y_pred)\n    \n\n    print(f'{model_name} Model:')\n    print(f'RMSLE: {rmsle}')\n    print(f'MAE: {mae}')\n    print(f'MSE: {mse}')\n    print(f'R²: {r2}')\n    print(f'Explained Variance Score: {explained_variance}')\n    print(f'MAPE: {mape}')\n    print('-' * 50)\n\n\"\"\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T20:35:43.022431Z","iopub.execute_input":"2024-12-30T20:35:43.022728Z","iopub.status.idle":"2024-12-30T20:35:43.029335Z","shell.execute_reply.started":"2024-12-30T20:35:43.022701Z","shell.execute_reply":"2024-12-30T20:35:43.028347Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T20:35:43.030399Z","iopub.execute_input":"2024-12-30T20:35:43.030765Z","iopub.status.idle":"2024-12-30T20:35:44.410801Z","shell.execute_reply.started":"2024-12-30T20:35:43.030728Z","shell.execute_reply":"2024-12-30T20:35:44.409725Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Optimizing best performer\n\"\"\"\nmodel = lgb.LGBMRegressor(\n        n_estimators=1000,\n        learning_rate=0.03,\n        num_leaves= 40,\n        subsample=0.7,\n        colsample_bytree=0.8,\n        random_state=42\n    )\n\n\nmodel.fit(X_train, y_train) \n\ny_pred = model.predict(X_test)  \n\n\nrmsle = np.sqrt(np.mean((y_pred - y_test) ** 2))\nmae = mean_absolute_error(y_test, y_pred)\nmse = mean_squared_error(y_test, y_pred)\nr2 = r2_score(y_test, y_pred)\nexplained_variance = explained_variance_score(y_test, y_pred)\nmape = mean_absolute_percentage_error(y_test, y_pred)\n\n\nprint(f'{model} Model:')\nprint(f'RMSLE: {rmsle}')\nprint(f'MAE: {mae}')\nprint(f'MSE: {mse}')\nprint(f'R²: {r2}')\nprint(f'Explained Variance Score: {explained_variance}')\nprint(f'MAPE: {mape}')\nprint('-' * 50)\n\n\n# LEARNING SPEED CHANGES\n#1 1.0517535157286826 .05 learning speed\n#2  1.0514007996250587 .03 learning speed\n#3 1.0514644204240065 .02 learning speed\n#4 1.0519312834025818 .06 learning speed\n\n# NUM LEAVES CHANGES SPEED .03\n#1 1.0516095839516597 20 leaves \n#2 1.0515960235091488 25 leaves\n#3 1.0514550756538577 30 leaves\n#4 1.0513642321895535 35 leaves\n#5 1.051315000544234 40 leaves\n#6  1.0514033860611454 45 leaves\n\n# SUBSAMPLE leaves 40\n#1. 1.051315000544234 .8 \n#2.  1.051315000544234 .9\n#3 1.051315000544234 .7\n\n#N estimator 1000 start\n#1 1.051466754003548\n\"\"\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T20:35:44.411895Z","iopub.execute_input":"2024-12-30T20:35:44.412288Z","iopub.status.idle":"2024-12-30T20:36:38.843449Z","shell.execute_reply.started":"2024-12-30T20:35:44.412249Z","shell.execute_reply":"2024-12-30T20:36:38.842331Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nfinal_model = lgb.LGBMRegressor(\n        n_estimators=1000,\n        learning_rate=0.03,\n        num_leaves= 40,\n        subsample=0.7,\n        colsample_bytree=0.8,\n        random_state=42\n    )\n\n\n\nfinal_model.fit(X, y)\ntest_predictions = final_model.predict(test)\nfinal_predictions = np.expm1(test_predictions)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T20:36:38.844379Z","iopub.execute_input":"2024-12-30T20:36:38.844699Z","iopub.status.idle":"2024-12-30T20:38:07.939808Z","shell.execute_reply.started":"2024-12-30T20:36:38.844671Z","shell.execute_reply":"2024-12-30T20:38:07.938709Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = pd.DataFrame({\n    'id': test_id, \n    'Premium Amount': final_predictions\n})\n\nsubmission.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T20:38:07.940943Z","iopub.execute_input":"2024-12-30T20:38:07.941259Z","iopub.status.idle":"2024-12-30T20:38:09.539677Z","shell.execute_reply.started":"2024-12-30T20:38:07.94123Z","shell.execute_reply":"2024-12-30T20:38:09.538474Z"}},"outputs":[],"execution_count":null}]}