{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30886,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfrom scipy.stats import normaltest\nimport scipy.stats as stats\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nfrom sklearn.impute import SimpleImputer\nfrom scipy.stats import zscore\nwarnings.simplefilter(action = \"ignore\", category = RuntimeWarning)\nfrom scipy.stats import skew\nfrom catboost import CatBoostRegressor, Pool\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_absolute_error, mean_squared_error\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T06:24:00.063135Z","iopub.execute_input":"2025-03-10T06:24:00.06345Z","iopub.status.idle":"2025-03-10T06:24:01.142912Z","shell.execute_reply.started":"2025-03-10T06:24:00.063424Z","shell.execute_reply":"2025-03-10T06:24:01.141938Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\ntrain_csv = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')\ntest_csv = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv')\nsample_submission_csv = pd.read_csv('/kaggle/input/playground-series-s4e12/sample_submission.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T06:24:03.152007Z","iopub.execute_input":"2025-03-10T06:24:03.152502Z","iopub.status.idle":"2025-03-10T06:24:10.377031Z","shell.execute_reply.started":"2025-03-10T06:24:03.15247Z","shell.execute_reply":"2025-03-10T06:24:10.376039Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**EDA**","metadata":{}},{"cell_type":"code","source":"train_csv.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T06:24:10.37842Z","iopub.execute_input":"2025-03-10T06:24:10.37883Z","iopub.status.idle":"2025-03-10T06:24:10.385798Z","shell.execute_reply.started":"2025-03-10T06:24:10.378796Z","shell.execute_reply":"2025-03-10T06:24:10.384854Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Checks for duplicate rows\nduplicates=train_csv.duplicated()\ntrain_csv[duplicates]\n#no duplicates so go ahead","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T06:24:11.769258Z","iopub.execute_input":"2025-03-10T06:24:11.76964Z","iopub.status.idle":"2025-03-10T06:24:13.416409Z","shell.execute_reply.started":"2025-03-10T06:24:11.769604Z","shell.execute_reply":"2025-03-10T06:24:13.415522Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.set_option('display.max_columns', None)\ntrain_csv.head(3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T06:24:13.41754Z","iopub.execute_input":"2025-03-10T06:24:13.417915Z","iopub.status.idle":"2025-03-10T06:24:13.437317Z","shell.execute_reply.started":"2025-03-10T06:24:13.417881Z","shell.execute_reply":"2025-03-10T06:24:13.436398Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_csv.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T06:24:19.053937Z","iopub.execute_input":"2025-03-10T06:24:19.054276Z","iopub.status.idle":"2025-03-10T06:24:19.740539Z","shell.execute_reply.started":"2025-03-10T06:24:19.054246Z","shell.execute_reply":"2025-03-10T06:24:19.739329Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Data Types looks fine**","metadata":{}},{"cell_type":"code","source":"#Imputation\nna_counts=train_csv.isna().sum()\nna_counts","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T06:24:26.066319Z","iopub.execute_input":"2025-03-10T06:24:26.066728Z","iopub.status.idle":"2025-03-10T06:24:26.738652Z","shell.execute_reply.started":"2025-03-10T06:24:26.066683Z","shell.execute_reply":"2025-03-10T06:24:26.737643Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_cols=train_csv.select_dtypes(include=[np.number])\n\n# Calculate skewness for each column\nskewness_results = num_cols.apply(lambda x: x.skew()).to_frame(name=\"Skewness\")\n\n# Classify Skewness Type\nskewness_results[\"Skewness Type\"] = skewness_results[\"Skewness\"].apply(\n    lambda x: \"Symmetric (Normal)\" if -0.5 <= x <= 0.5 else \n              \"Moderate Skew\" if -1 <= x < -0.5 or 0.5 < x <= 1 else \n              \"Highly Skewed\"\n)\n\n# Display results\nprint(skewness_results)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T06:24:26.74013Z","iopub.execute_input":"2025-03-10T06:24:26.740526Z","iopub.status.idle":"2025-03-10T06:24:26.950008Z","shell.execute_reply.started":"2025-03-10T06:24:26.740486Z","shell.execute_reply":"2025-03-10T06:24:26.94918Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Categorize columns based on skewness for outliers\nnormal=list(skewness_results[skewness_results['Skewness Type']==\"Symmetric (Normal)\"].index)\nskewed=list(skewness_results[skewness_results['Skewness Type']!=\"Symmetric (Normal)\"].index)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T06:24:29.615988Z","iopub.execute_input":"2025-03-10T06:24:29.616311Z","iopub.status.idle":"2025-03-10T06:24:29.623808Z","shell.execute_reply.started":"2025-03-10T06:24:29.616285Z","shell.execute_reply":"2025-03-10T06:24:29.622828Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Data has columns both skewed and Normal. Opting for Mean imputation where cols are normal and median imputation if skewed**","metadata":{}},{"cell_type":"code","source":"def impute_based_on_skewness(data):\n    for col in num_cols.columns:\n        if data[col].isnull().sum() > 0:  # Apply imputation only if there are missing values\n            col_skewness = skew(data[col].dropna())  # Compute skewness ignoring NaNs\n            \n            # Normal Distribution (Mean Imputation)\n            if -0.5 <= col_skewness <= 0.5:\n                imputer = SimpleImputer(strategy=\"mean\")\n                data.loc[:, col] = imputer.fit_transform(data[[col]])                \n\n            # Skewed Distribution (Median Imputation)\n            else:\n                imputer = SimpleImputer(strategy=\"median\")\n                data.loc[:, col] = imputer.fit_transform(data[[col]])  # Use 2D array\n    return data\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T06:24:40.170543Z","iopub.execute_input":"2025-03-10T06:24:40.170901Z","iopub.status.idle":"2025-03-10T06:24:40.176624Z","shell.execute_reply.started":"2025-03-10T06:24:40.170873Z","shell.execute_reply":"2025-03-10T06:24:40.175469Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_csv=pd.DataFrame(impute_based_on_skewness(train_csv.copy()))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T06:24:41.070156Z","iopub.execute_input":"2025-03-10T06:24:41.070478Z","iopub.status.idle":"2025-03-10T06:24:42.241005Z","shell.execute_reply.started":"2025-03-10T06:24:41.070452Z","shell.execute_reply":"2025-03-10T06:24:42.23994Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Impute missing categorical values with mode (most frequent value)\ncategorical_cols = train_csv.select_dtypes(include=[\"object\"]).columns\n\n# Apply imputation\nimputer = SimpleImputer(strategy=\"most_frequent\")\ntrain_csv[categorical_cols] = imputer.fit_transform(train_csv[categorical_cols])\n\nprint(\"Categorical values imputed using most frequent strategy!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T06:24:43.434295Z","iopub.execute_input":"2025-03-10T06:24:43.434693Z","iopub.status.idle":"2025-03-10T06:24:46.127946Z","shell.execute_reply.started":"2025-03-10T06:24:43.434656Z","shell.execute_reply":"2025-03-10T06:24:46.127035Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Imputation check\nna_counts=train_csv.isna().sum()\nna_counts","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T06:24:46.129104Z","iopub.execute_input":"2025-03-10T06:24:46.129434Z","iopub.status.idle":"2025-03-10T06:24:46.804032Z","shell.execute_reply.started":"2025-03-10T06:24:46.129397Z","shell.execute_reply":"2025-03-10T06:24:46.802907Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Metric\tHow to Use for Outliers?\n\nmean\tIf much larger/smaller than median (50%), data is skewed, possible outliers.\n\nstd (Standard Deviation)\tIf very high, the column has high variability (possible extreme values).\n\nmin & max\tIf max is far from Q3 (75%) or min is far from Q1 (25%), extreme values exist.\n\n25% (Q1) & 75% (Q3)\tUse to compute IQR and check for values outside 1.5× IQR range.\n\n50% (Median)\tIf very different from mean, the data is skewed (potential outliers).","metadata":{}},{"cell_type":"code","source":"pd.options.display.float_format = '{:.2f}'.format\ntrain_csv.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T06:24:46.805518Z","iopub.execute_input":"2025-03-10T06:24:46.80592Z","iopub.status.idle":"2025-03-10T06:24:47.350667Z","shell.execute_reply.started":"2025-03-10T06:24:46.805883Z","shell.execute_reply":"2025-03-10T06:24:47.349651Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"✔ IQR for skewed columns (better for non-normal data)\n\n✔ Z-score for normal columns (better for normal distributions)","metadata":{"execution":{"iopub.status.busy":"2025-02-25T06:14:08.287926Z","iopub.execute_input":"2025-02-25T06:14:08.28832Z","iopub.status.idle":"2025-02-25T06:14:08.295365Z","shell.execute_reply.started":"2025-02-25T06:14:08.288292Z","shell.execute_reply":"2025-02-25T06:14:08.293573Z"}}},{"cell_type":"code","source":"\n# Function to detect outliers using Z-score\ndef detect_outliers_zscore(data, threshold=3):\n    outlier_summary = {}\n    \n    for col in data.columns:\n        z_scores = np.abs(zscore(data[col].dropna()))  # Compute absolute Z-scores\n        outlier_count = (z_scores > threshold).sum()  # Count values above threshold\n\n        outlier_summary[col] = {\n            \"Total Outliers\": outlier_count,\n            \"Percentage of Outliers\": round((outlier_count / len(data)) * 100, 2)\n        }\n\n    return pd.DataFrame(outlier_summary).T\n\n# Run the function\noutlier_results_z = detect_outliers_zscore(train_csv[normal])\n\n# Display results\nprint(outlier_results_z)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T06:24:50.426153Z","iopub.execute_input":"2025-03-10T06:24:50.426473Z","iopub.status.idle":"2025-03-10T06:24:50.659764Z","shell.execute_reply.started":"2025-03-10T06:24:50.426446Z","shell.execute_reply":"2025-03-10T06:24:50.658909Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**No outliers in normal data**","metadata":{}},{"cell_type":"code","source":"\n# Function to detect outliers using IQR\ndef detect_outliers_iqr(data):\n    outlier_summary = {}\n    \n    for col in data.columns:\n        Q1 = data[col].quantile(0.25)  # 25th percentile\n        Q3 = data[col].quantile(0.75)  # 75th percentile\n        IQR = Q3 - Q1  # Interquartile range\n\n        lower_bound = Q1 - 1.5 * IQR\n        upper_bound = Q3 + 1.5 * IQR\n\n        outliers = data[(data[col] < lower_bound) | (data[col] > upper_bound)][col]\n        outlier_count = outliers.count()\n        \n        outlier_summary[col] = {\n            \"Total Outliers\": outlier_count,\n            \"Percentage of Outliers\": round((outlier_count / len(data)) * 100, 2)\n        }\n\n    return pd.DataFrame(outlier_summary).T\n\n# Run the function\noutlier_results = detect_outliers_iqr(train_csv[skewed])\n\n# Display the result\nprint(outlier_results)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T06:24:53.112362Z","iopub.execute_input":"2025-03-10T06:24:53.11274Z","iopub.status.idle":"2025-03-10T06:24:53.287703Z","shell.execute_reply.started":"2025-03-10T06:24:53.112707Z","shell.execute_reply":"2025-03-10T06:24:53.286661Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"> Annual Income and premium amount are important cols . Risk of missing information is high if removed. so log tranform the 2 cols\n","metadata":{}},{"cell_type":"code","source":"\n\n# Apply log transformation only on skewed columns\nskewed_cols = [\"Annual Income\", \"Previous Claims\",\"Premium Amount\"]  # Modify based on data distribution\n\ntransformed_data = train_csv.copy()\n\nfor col in skewed_cols:\n    if col in transformed_data.columns:  # Ensure the column exists\n        transformed_data[col] = np.log1p(transformed_data[col])  # log1p avoids log(0) issues\n\n\ntrain_csv=transformed_data.copy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T06:24:55.941748Z","iopub.execute_input":"2025-03-10T06:24:55.942097Z","iopub.status.idle":"2025-03-10T06:24:56.821872Z","shell.execute_reply.started":"2025-03-10T06:24:55.942069Z","shell.execute_reply":"2025-03-10T06:24:56.821029Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_csv_=train_csv.copy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T06:24:57.892389Z","iopub.execute_input":"2025-03-10T06:24:57.892751Z","iopub.status.idle":"2025-03-10T06:24:58.042955Z","shell.execute_reply.started":"2025-03-10T06:24:57.892717Z","shell.execute_reply":"2025-03-10T06:24:58.042126Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_csv['Policy Start Date']=pd.to_datetime(train_csv['Policy Start Date'])\ntrain_csv['Year']=train_csv['Policy Start Date'].dt.year\ntrain_csv['Day']=train_csv['Policy Start Date'].dt.day\ntrain_csv['Month']=train_csv['Policy Start Date'].dt.month\ntrain_csv.drop(columns=['id','Policy Start Date'],inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T06:24:58.28479Z","iopub.execute_input":"2025-03-10T06:24:58.285144Z","iopub.status.idle":"2025-03-10T06:24:59.007487Z","shell.execute_reply.started":"2025-03-10T06:24:58.285112Z","shell.execute_reply":"2025-03-10T06:24:59.006637Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = train_csv.drop(columns=[\"Premium Amount\"])  # Replace \"target\" with actual target column name\ny = train_csv[\"Premium Amount\"]\n\n# Identify categorical columns (CatBoost expects string names or indices)\ncategorical_cols = X.select_dtypes(include=[\"object\", \"category\"]).columns.tolist()\n\n# Initialize CatBoost model\nmodel = CatBoostRegressor(\n    iterations=500,        # Number of boosting rounds\n    learning_rate=0.05,    # Step size for learning\n    depth=6,               # Tree depth\n    cat_features=categorical_cols,  # Let CatBoost handle categorical data\n    loss_function=\"RMSE\",  # Root Mean Squared Error (good for regression)\n    eval_metric=\"MAE\",     # Mean Absolute Error for evaluation\n    verbose=100\n)\n# Train model (CatBoost handles encoding internally)\nmodel.fit(X, y)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T06:25:00.325703Z","iopub.execute_input":"2025-03-10T06:25:00.326058Z","iopub.status.idle":"2025-03-10T06:32:06.211677Z","shell.execute_reply.started":"2025-03-10T06:25:00.326029Z","shell.execute_reply":"2025-03-10T06:32:06.210796Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T06:33:09.34412Z","iopub.execute_input":"2025-03-10T06:33:09.344453Z","iopub.status.idle":"2025-03-10T06:33:09.348783Z","shell.execute_reply.started":"2025-03-10T06:33:09.344426Z","shell.execute_reply":"2025-03-10T06:33:09.347703Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred = model.predict(X)\n\n# Evaluation Metrics\nmae = mean_absolute_error(y, y_pred)\nmse = mean_squared_error(y, y_pred)\nrmse = np.sqrt(mse)\n\nprint(f\"Mean Absolute Error (MAE): {mae:.4f}\")\nprint(f\"Root Mean Squared Error (RMSE): {rmse:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T06:33:28.675064Z","iopub.execute_input":"2025-03-10T06:33:28.675429Z","iopub.status.idle":"2025-03-10T06:33:31.054604Z","shell.execute_reply.started":"2025-03-10T06:33:28.675395Z","shell.execute_reply":"2025-03-10T06:33:31.05331Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mae = mean_absolute_error(y, y_pred)\nregression_accuracy = 1 - (mae / np.mean(y))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T06:34:20.811987Z","iopub.execute_input":"2025-03-10T06:34:20.812336Z","iopub.status.idle":"2025-03-10T06:34:20.832926Z","shell.execute_reply.started":"2025-03-10T06:34:20.812308Z","shell.execute_reply":"2025-03-10T06:34:20.831781Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"regression_accuracy","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T06:34:32.316335Z","iopub.execute_input":"2025-03-10T06:34:32.316745Z","iopub.status.idle":"2025-03-10T06:34:32.322419Z","shell.execute_reply.started":"2025-03-10T06:34:32.316705Z","shell.execute_reply":"2025-03-10T06:34:32.321498Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submit = pd.read_csv(\"/kaggle/input/playground-series-s4e12/sample_submission.csv\")\nsubmit[\"Premium Amount\"] = np.exp( y )-1\nsubmit.to_csv(\"submission.csv\",index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}