{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30918,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import libraries and initial data analysis","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport scipy.stats as stats\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:11:59.17391Z","iopub.execute_input":"2025-03-18T19:11:59.174119Z","iopub.status.idle":"2025-03-18T19:11:59.958856Z","shell.execute_reply.started":"2025-03-18T19:11:59.174097Z","shell.execute_reply":"2025-03-18T19:11:59.957966Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')\ndf_test = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:11:59.959642Z","iopub.execute_input":"2025-03-18T19:11:59.960013Z","iopub.status.idle":"2025-03-18T19:12:05.987658Z","shell.execute_reply.started":"2025-03-18T19:11:59.95998Z","shell.execute_reply":"2025-03-18T19:12:05.986745Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dataset = [\n    (df_train, \"train\"),\n    (df_test, \"test\")\n]\n\nfor df, name in dataset:\n    print(f\"There is {df.shape[0]} rows and {df.shape[1]} columns in the {name} dataset.\")\n    sum_data_duplicates = df.duplicated().sum()\n    print(f\"Duplicated fields in {name} dataset: {sum_data_duplicates}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:12:05.99001Z","iopub.execute_input":"2025-03-18T19:12:05.990432Z","iopub.status.idle":"2025-03-18T19:12:08.120737Z","shell.execute_reply.started":"2025-03-18T19:12:05.990407Z","shell.execute_reply":"2025-03-18T19:12:08.119984Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:12:08.121769Z","iopub.execute_input":"2025-03-18T19:12:08.121973Z","iopub.status.idle":"2025-03-18T19:12:08.718253Z","shell.execute_reply.started":"2025-03-18T19:12:08.121954Z","shell.execute_reply":"2025-03-18T19:12:08.717336Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:12:08.718989Z","iopub.execute_input":"2025-03-18T19:12:08.719239Z","iopub.status.idle":"2025-03-18T19:12:09.119009Z","shell.execute_reply.started":"2025-03-18T19:12:08.719218Z","shell.execute_reply":"2025-03-18T19:12:09.118105Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:12:09.119947Z","iopub.execute_input":"2025-03-18T19:12:09.120286Z","iopub.status.idle":"2025-03-18T19:12:09.146841Z","shell.execute_reply.started":"2025-03-18T19:12:09.120256Z","shell.execute_reply":"2025-03-18T19:12:09.146008Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:12:09.147726Z","iopub.execute_input":"2025-03-18T19:12:09.147956Z","iopub.status.idle":"2025-03-18T19:12:09.165526Z","shell.execute_reply.started":"2025-03-18T19:12:09.147926Z","shell.execute_reply":"2025-03-18T19:12:09.164626Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for df, name in dataset:\n    print(f\"\\n--- {name.upper()} Dataset ---\")\n    total_rows = df.shape[0]\n    null_counts = df.isnull().sum()\n    null_columns = null_counts[null_counts > 0]\n    \n    if not null_columns.empty:\n        print(\"Columns with null values (count | %):\")\n        # Sort columns by null count (or percentage) in descending order\n        sorted_null_columns = null_columns.sort_values(ascending=True)\n        for col, count in sorted_null_columns.items():\n            percent = (count / total_rows) * 100\n            print(f\"- {col}: {count} nulls ({percent:.2f}%)\")\n    else:\n        print(\"No columns with null values.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:12:09.166402Z","iopub.execute_input":"2025-03-18T19:12:09.166606Z","iopub.status.idle":"2025-03-18T19:12:10.165341Z","shell.execute_reply.started":"2025-03-18T19:12:09.166588Z","shell.execute_reply":"2025-03-18T19:12:10.164576Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"summary_dict = {col: df[col].describe() for col in null_columns.index}\n\n# To display the summaries:\nfor col, summary in summary_dict.items():\n    print(f\"Summary for column: {col}\")\n    print(summary)\n    print(\"\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:12:10.166236Z","iopub.execute_input":"2025-03-18T19:12:10.166483Z","iopub.status.idle":"2025-03-18T19:12:10.714051Z","shell.execute_reply.started":"2025-03-18T19:12:10.166451Z","shell.execute_reply":"2025-03-18T19:12:10.713119Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Exploratory data analysis","metadata":{}},{"cell_type":"markdown","source":"## Target values","metadata":{}},{"cell_type":"code","source":"plt.hist(df_train[\"Premium Amount\"], bins=50, edgecolor='black')  # edgecolor adds bar borders\nplt.title('Distribution of Premium Amounts')  # Fixed title to match the data\nplt.xlabel('Premium Amount')\nplt.ylabel('Frequency')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:12:10.715069Z","iopub.execute_input":"2025-03-18T19:12:10.715422Z","iopub.status.idle":"2025-03-18T19:12:11.021374Z","shell.execute_reply.started":"2025-03-18T19:12:10.715387Z","shell.execute_reply":"2025-03-18T19:12:11.020419Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stats.probplot(df_train[\"Premium Amount\"], dist=\"norm\", plot=plt)\nplt.title('Normal Q-Q plot Sale Prices')\nplt.xlabel('Theoretical quantiles')\nplt.ylabel('Ordered Values')\nplt.grid(True)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:12:11.022287Z","iopub.execute_input":"2025-03-18T19:12:11.022589Z","iopub.status.idle":"2025-03-18T19:12:13.534422Z","shell.execute_reply.started":"2025-03-18T19:12:11.022553Z","shell.execute_reply":"2025-03-18T19:12:13.533505Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import PowerTransformer, QuantileTransformer\n\n# Initialize transformers\ntransformers = {\n    'Box-Cox': PowerTransformer(method='box-cox', standardize=True),\n    'Yeo-Johnson': PowerTransformer(method='yeo-johnson', standardize=True),\n    'Quantile': QuantileTransformer(output_distribution='normal')\n}\n\n# Apply transformations and store results\ntransformed_data = {\n    'Original': df_train[\"Premium Amount\"]\n}\nfor name, transformer in transformers.items():\n    transformed_data[name] = transformer.fit_transform(df_train[[\"Premium Amount\"]]).flatten()\n\n# Plot histograms\nplt.figure(figsize=(18, 6))\nfor i, (name, data) in enumerate(transformed_data.items(), 1):\n    plt.subplot(1, 4, i)\n    plt.hist(data, bins=50, edgecolor='black')\n    plt.title(f\"{name} Data\" if name == 'Original' else f\"{name} Transformed\")\n    plt.xlabel(\"Premium Amount\" if name == 'Original' else \"Transformed Value\")\n    plt.ylabel(\"Frequency\")\nplt.tight_layout()\nplt.show()\n\n# Plot Q-Q plots\nplt.figure(figsize=(18, 6))\nfor i, (name, data) in enumerate(transformed_data.items(), 1):\n    plt.subplot(1, 4, i)\n    stats.probplot(data, dist=\"norm\", plot=plt)\n    plt.title(f'Q-Q Plot: {name} Data' if name == 'Original' else f'Q-Q Plot: {name} Transformed')\n    plt.xlabel('Theoretical Quantiles')\n    plt.ylabel('Ordered Values')\n    plt.grid(True)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:12:13.536888Z","iopub.execute_input":"2025-03-18T19:12:13.537112Z","iopub.status.idle":"2025-03-18T19:12:31.960979Z","shell.execute_reply.started":"2025-03-18T19:12:13.537092Z","shell.execute_reply":"2025-03-18T19:12:31.960041Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train[\"Premium Amount\"].describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:12:31.962362Z","iopub.execute_input":"2025-03-18T19:12:31.962728Z","iopub.status.idle":"2025-03-18T19:12:32.013655Z","shell.execute_reply.started":"2025-03-18T19:12:31.962692Z","shell.execute_reply":"2025-03-18T19:12:32.012768Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Numerical values","metadata":{}},{"cell_type":"code","source":"columns_numerical_values = df_train.select_dtypes(\"number\").drop(columns=[\"Premium Amount\", \"id\"])\ncolumns_numerical_values.count()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:12:32.014499Z","iopub.execute_input":"2025-03-18T19:12:32.014726Z","iopub.status.idle":"2025-03-18T19:12:32.1176Z","shell.execute_reply.started":"2025-03-18T19:12:32.014707Z","shell.execute_reply":"2025-03-18T19:12:32.116784Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col in columns_numerical_values.columns:\n    plt.figure(figsize=(12, 5))\n    \n    # Left subplot: Histogram\n    plt.subplot(1, 2, 1)\n    plt.hist(columns_numerical_values[col].dropna(), bins=30, color='skyblue', edgecolor='black')\n    plt.title(f\"Histogram for {col}\")\n    plt.xlabel(col)\n    plt.ylabel(\"Frequency\")\n    \n    # Right subplot: Boxplot with colors\n    plt.subplot(1, 2, 2)\n    bp = plt.boxplot(columns_numerical_values[col].dropna(), vert=False, patch_artist=True)\n    plt.title(f\"Boxplot for {col}\")\n    plt.xlabel(col)\n    \n    # Customize the boxplot colors\n    for box in bp['boxes']:\n        box.set(facecolor='lightgreen', color='darkgreen')\n    for whisker in bp['whiskers']:\n        whisker.set(color='blue', linestyle='--')\n    for cap in bp['caps']:\n        cap.set(color='red')\n    for median in bp['medians']:\n        median.set(color='orange')\n    \n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:12:32.118505Z","iopub.execute_input":"2025-03-18T19:12:32.118828Z","iopub.status.idle":"2025-03-18T19:12:35.48348Z","shell.execute_reply.started":"2025-03-18T19:12:32.118794Z","shell.execute_reply":"2025-03-18T19:12:35.482549Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Categorical values","metadata":{}},{"cell_type":"code","source":"columns_object_values = df_train.select_dtypes(\"object\")\ncolumns_object_values.count()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:12:35.484319Z","iopub.execute_input":"2025-03-18T19:12:35.484542Z","iopub.status.idle":"2025-03-18T19:12:36.078872Z","shell.execute_reply.started":"2025-03-18T19:12:35.484522Z","shell.execute_reply":"2025-03-18T19:12:36.078008Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col in columns_object_values.columns:\n    # Get value counts for each category\n    counts = columns_object_values[col].value_counts()\n    \n    # Plot a bar chart of the value counts\n    plt.figure(figsize=(10, 5))\n    counts.plot(kind=\"bar\", color='skyblue', edgecolor='black')\n    plt.title(f\"Distribution of {col}\")\n    plt.xlabel(col)\n    plt.ylabel(\"Frequency\")\n    plt.xticks(rotation=45)\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-18T19:12:36.079665Z","iopub.execute_input":"2025-03-18T19:12:36.07992Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train vs Test set comparison","metadata":{}}]}