{"metadata":{"kernelspec":{"name":"ir","display_name":"R","language":"R"},"language_info":{"name":"R","codemirror_mode":"r","pygments_lexer":"r","mimetype":"text/x-r-source","file_extension":".r","version":"4.4.0"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30749,"isInternetEnabled":true,"language":"r","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import libraries and data","metadata":{}},{"cell_type":"code","source":"# Import Libraries\nsuppressPackageStartupMessages({\n\nlibrary(tidyverse)  # Collection of packages including dplyr, ggplot2\nlibrary(snakecase)  # Converts strings to snake_case format\nlibrary(lubridate)  # Simplifies date and time handling\nlibrary(naniar)     # Tools for exploring and handling missing data\nlibrary(janitor)    # Data cleaning and examination functions\nlibrary(caret)      # Machine learning training and evaluation\nlibrary(lightgbm)   # Gradient boosting\n\n})","metadata":{"_uuid":"051d70d956493feee0c6d64651c6a088724dca2a","_execution_state":"idle","trusted":true,"execution":{"iopub.status.busy":"2024-12-28T10:23:26.528799Z","iopub.execute_input":"2024-12-28T10:23:26.531185Z","iopub.status.idle":"2024-12-28T10:23:29.450747Z","shell.execute_reply":"2024-12-28T10:23:29.448937Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Import data \ntrain <- read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\")\ntest <- read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T10:23:34.378292Z","iopub.execute_input":"2024-12-28T10:23:34.409976Z","iopub.status.idle":"2024-12-28T10:23:42.633779Z","shell.execute_reply":"2024-12-28T10:23:42.631681Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Explore and clean data","metadata":{}},{"cell_type":"code","source":"# Inspect train set\nsummary(train)\n\n# Print the first few rows \nhead(train)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Inspect test set\nsummary(test)\n\n# Print the first few rows\nhead(test)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Convert column names and apostrophes","metadata":{}},{"cell_type":"code","source":"# Function to clean and transform dataframe\nclean_and_transform <- function(df) {\n  # Convert column names to snake_case \n  colnames(df) <- snakecase::to_snake_case(colnames(df))\n  \n  # Clean and transform columns with vectorized operations\n  df <- df %>%\n    dplyr::mutate(\n      education_level = gsub(\"'\", \"\", education_level),\n      education_level = snakecase::to_snake_case(education_level),\n      occupation = snakecase::to_snake_case(occupation)\n    )\n  \n  return(df)\n}\n\n# Apply transformations to train and test dataframe\ntrain <- clean_and_transform(train)\ntest <- clean_and_transform(test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T10:23:48.731324Z","iopub.execute_input":"2024-12-28T10:23:48.733149Z","iopub.status.idle":"2024-12-28T10:26:57.800154Z","shell.execute_reply":"2024-12-28T10:26:57.798183Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Visualization ","metadata":{}},{"cell_type":"code","source":"# List of numerical and categorical columns\nnumerical_cols <- c(\"age\", \"annual_income\", \"health_score\", \"vehicle_age\", \n                    \"credit_score\", \"insurance_duration\", \"number_of_dependents\", \"previous_claims\")\ncategorical_cols <- c(\"gender\", \"marital_status\", \"education_level\", \"occupation\", \n                      \"location\", \"policy_type\", \"customer_feedback\", \"smoking_status\", \n                      \"exercise_frequency\", \"property_type\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T10:28:09.199523Z","iopub.execute_input":"2024-12-28T10:28:09.201529Z","iopub.status.idle":"2024-12-28T10:28:09.218988Z","shell.execute_reply":"2024-12-28T10:28:09.216991Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Function to plot pie charts for proportions\nplot_categorical_proportions <- function(df, columns) {\n  for (col in columns) {\n    p <- df %>%\n      count(!!sym(col)) %>%\n      ggplot(aes(x = \"\", y = n, fill = !!sym(col))) +\n      geom_bar(stat = \"identity\", width = 1) +\n      coord_polar(\"y\", start = 0) +\n      labs(title = paste(\"Proportion of\", col), fill = col) +\n      theme_void() +\n      theme(legend.position = \"right\")\n    print(p) # print the ggplot object\n  }\n}\n\n# Plot proportions\nplot_categorical_proportions(train, categorical_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T10:28:12.504481Z","iopub.execute_input":"2024-12-28T10:28:12.506197Z","iopub.status.idle":"2024-12-28T10:28:15.54986Z","shell.execute_reply":"2024-12-28T10:28:15.546367Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Function to create histogram plot\nplot_numeric_distributions <- function(data, title) {\n  numeric_cols <- data %>% select(where(is.numeric))\n  numeric_long <- numeric_cols %>%\n    pivot_longer(cols = everything(), names_to = \"variable\", values_to = \"value\")\n  \n  # Remove non-finite values\n  numeric_long <- numeric_long %>% filter(is.finite(value))\n  \n  # Convert to long format for ggplot\n  ggplot(numeric_long, aes(x = value)) +\n    geom_histogram(bins = 30, fill = \"skyblue\", color = \"black\") +\n    facet_wrap(~variable, scales = \"free\", ncol = 3) +\n    labs(title = title, x = \"Value\", y = \"Frequency\") +\n    theme_minimal() +\n    scale_x_continuous(limits = function(x) c(min(x), max(x)))\n}\n\n# Call the function for both datasets\nplot_numeric_distributions(train, \"Distributions of Numeric Columns (Train)\")\nplot_numeric_distributions(test, \"Distributions of Numeric Columns (Test)\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T10:28:24.104896Z","iopub.execute_input":"2024-12-28T10:28:24.10646Z","iopub.status.idle":"2024-12-28T10:28:47.755554Z","shell.execute_reply":"2024-12-28T10:28:47.752793Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Function to create boxplots for numeric columns\nplot_numeric_boxplots <- function(data, title) {\n  # Select only numeric columns\n  numeric_cols <- data %>% select(where(is.numeric))\n  \n  # Convert to long format for ggplot\n  numeric_long <- numeric_cols %>%\n    pivot_longer(cols = everything(), names_to = \"variable\", values_to = \"value\")\n\n  # Remove non-finite values\n  numeric_long <- numeric_long %>% filter(is.finite(value))\n    \n  # Create the boxplot\n  ggplot(numeric_long, aes(x = variable, y = value)) +\n    geom_boxplot(fill = \"skyblue\", outlier.color = \"red\", outlier.shape = 1) +\n    coord_flip() +\n    labs(title = title,\n         x = \"Variable\", y = \"Value\") +\n    theme_minimal()\n}\n\n# Plot for train dataframe\nplot_numeric_boxplots(train, \"Boxplots of Numeric Columns in Train Data\")\n\n# Plot for test dataframe\nplot_numeric_boxplots(test, \"Boxplots of Numeric Columns in Test Data\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T10:29:01.466814Z","iopub.execute_input":"2024-12-28T10:29:01.468417Z","iopub.status.idle":"2024-12-28T10:29:27.999115Z","shell.execute_reply":"2024-12-28T10:29:27.996112Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create boxplots for specified variables\ncreate_boxplots <- function(data, variables, dataset_name) {\n  par(mfrow = c(1, length(variables))) # Adjust layout based on number of variables\n  for (var in variables) {\n    boxplot(data[[var]], \n            main = paste(dataset_name, var), \n            ylab = \"Value\", \n            col = \"skyblue\",\n            outline = TRUE) # Highlight outliers\n  }\n  par(mfrow = c(1,1)) # Reset layout to default\n}\n\n# Variables to plot\nvariables_to_plot <- c(\"annual_income\", \"previous_claims\")\n\n# Call the function for train and test datasets\ncreate_boxplots(train, variables_to_plot, \"Train\")\ncreate_boxplots(test, variables_to_plot, \"Test\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Transform Features","metadata":{}},{"cell_type":"markdown","source":"## Impute NA values","metadata":{}},{"cell_type":"code","source":"# Impute numerical columns with mean\nfor (col in numerical_cols) {\n  mean_value <- mean(train[[col]], na.rm = TRUE)\n  train[[col]][is.na(train[[col]])] <- mean_value\n  test[[col]][is.na(test[[col]])] <- mean_value\n}\n\n# Replace missing values in categorical columns with unknown \nfor (col in categorical_cols) {\n  train[[col]][is.na(train[[col]])] <- \"unknown\"\n  test[[col]][is.na(test[[col]])] <- \"unknown\"\n}\n\n# Impute datetime column with the median date\nmedian_date <- median(as.POSIXct(train$policy_start_date, format = \"%Y-%m-%d %H:%M:%S\"), na.rm = TRUE)\ntrain$policy_start_date[is.na(train$policy_start_date)] <- as.character(median_date)\ntest$policy_start_date[is.na(test$policy_start_date)] <- as.character(median_date)\n\n# Verify no missing values remain\ncolSums(is.na(train))\ncolSums(is.na(test))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T10:29:46.018947Z","iopub.execute_input":"2024-12-28T10:29:46.020595Z","iopub.status.idle":"2024-12-28T10:29:46.78203Z","shell.execute_reply":"2024-12-28T10:29:46.779555Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Feature Scaling","metadata":{}},{"cell_type":"code","source":"# Normalize continuous features to improve model convergence \nmin_max_normalize <- function(x) {\n  (x - min(x)) / (max(x) - min(x))\n}\n\n# Normalize features in train dataset\ntrain$annual_income_normalized <- min_max_normalize(train$annual_income)\ntrain$credit_score_normalized <- min_max_normalize(train$credit_score)\ntrain$health_score_normalized <- min_max_normalize(train$health_score)\n\n# Normalize features in test dataset\n# Important: Use min and max values from train dataset for test dataset\ntest$annual_income_normalized <- (test$annual_income - min(train$annual_income)) / \n  (max(train$annual_income) - min(train$annual_income))\ntest$credit_score_normalized <- (test$credit_score - min(train$credit_score)) / \n  (max(train$credit_score) - min(train$credit_score))\ntest$health_score_normalized <- (test$health_score - min(train$health_score)) / \n  (max(train$health_score) - min(train$health_score))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T10:29:51.696025Z","iopub.execute_input":"2024-12-28T10:29:51.697548Z","iopub.status.idle":"2024-12-28T10:29:51.78601Z","shell.execute_reply":"2024-12-28T10:29:51.784209Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Handle Outliers","metadata":{}},{"cell_type":"code","source":"# Function to cap outliers\ncap_outliers <- function(data, column, lower_bound, upper_bound) {\n  data[[column]] <- pmin(pmax(data[[column]], lower_bound), upper_bound)\n  return(data)\n}\n\n# Calculate bounds for previous_claims using IQR method\nbounds_previous_claims <- quantile(train$previous_claims, probs = c(0.25, 0.75))\niqr_previous_claims <- bounds_previous_claims[2] - bounds_previous_claims[1]\nbounds_previous_claims <- c(bounds_previous_claims[1] - 1.5 * iqr_previous_claims,\n                             bounds_previous_claims[2] + 1.5 * iqr_previous_claims)\n\n# Calculate bounds for annual_income \nbounds_annual_income <- quantile(train$annual_income, probs = c(0.25, 0.75))\niqr_annual_income <- bounds_annual_income[2] - bounds_annual_income[1]\nbounds_annual_income <- c(bounds_annual_income[1] - 1.5 * iqr_annual_income,\n                           bounds_annual_income[2] + 1.5 * iqr_annual_income)\n\n# Handle outliers in train dataset\ntrain <- cap_outliers(train, \"previous_claims\", bounds_previous_claims[1], bounds_previous_claims[2])\ntrain <- cap_outliers(train, \"annual_income\", bounds_annual_income[1], bounds_annual_income[2])\n\n# Handle outliers in test dataset\ntest <- cap_outliers(test, \"previous_claims\", bounds_previous_claims[1], bounds_previous_claims[2])\ntest <- cap_outliers(test, \"annual_income\", bounds_annual_income[1], bounds_annual_income[2])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T10:29:55.713247Z","iopub.execute_input":"2024-12-28T10:29:55.714858Z","iopub.status.idle":"2024-12-28T10:29:55.861212Z","shell.execute_reply":"2024-12-28T10:29:55.859445Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Modeling","metadata":{}},{"cell_type":"code","source":"# Set seed for reproducibility\nset.seed(123)\n\n# Splitting the data into train (80%) and validation (20%)\ntrain_index <- createDataPartition(train$premium_amount, p = 0.8, list = FALSE)\ntrain_set <- train[train_index, ]\nvalid_set <- train[-train_index, ]\n\n# Log transform the target variable\ntrain_label <- log1p(train_set$premium_amount)\nvalid_label <- log1p(valid_set$premium_amount)\n\n# Exclude unnecessary columns\nexclude_cols <- c(\"id\", \"policy_start_date\", \"premium_amount\")\ntrain_set <- train_set %>% select(-all_of(exclude_cols))\nvalid_set <- valid_set %>% select(-all_of(exclude_cols))\n\n# Function to convert categorical variables to numeric encoding\nconvert_categorical_to_numeric <- function(df1, df2) {\n  # Loop through all columns in df1\n  for (col_name in names(df1)) {\n    # Check if the column in df1 is categorical (character or factor)\n    if (is.character(df1[[col_name]]) || is.factor(df1[[col_name]])) {\n      # Combine unique levels from both dataframes to ensure consistency\n      combined_levels <- unique(c(df1[[col_name]], df2[[col_name]]))\n      \n      # Convert df1 and df2 columns to numeric based on the combined levels\n      df1[[col_name]] <- as.numeric(factor(df1[[col_name]], levels = combined_levels))\n      df2[[col_name]] <- as.numeric(factor(df2[[col_name]], levels = combined_levels))\n    }\n  }\n  # Return the updated dataframes as a list\n  return(list(df1 = df1, df2 = df2))\n}\n\n# Apply the function to train and validation datasets\nconverted_data <- convert_categorical_to_numeric(train_set, valid_set)\n\n# Update train and validation datasets with the converted values\ntrain_set <- converted_data$df1\nvalid_set <- converted_data$df2\n\n\n# Convert data to matrices\ntrain_matrix <- as.matrix(train_set)\nvalid_matrix <- as.matrix(valid_set)\n\n# Check for NA values\nif (any(is.na(train_matrix)) || any(is.na(valid_matrix))) {\n  stop(\"NA values found in matrices.\")\n}\n\n# Create LightGBM datasets\nlgb_train <- lgb.Dataset(data = train_matrix, label = train_label)\nlgb_valid <- lgb.Dataset(data = valid_matrix, label = valid_label)\n\n# Train the LightGBM model with validation\nparams <- list(\n  objective = \"regression\",\n  metric = \"rmse\",\n  boosting_type = \"gbdt\",\n  learning_rate = 0.05,  \n  num_leaves = 100,\n  max_depth = 10,\n  min_data_in_leaf = 20  # Prevent overfitting\n)\n\nlgb_model <- lgb.train(\n  params = params,\n  data = lgb_train,\n  valids = list(validation = lgb_valid),\n  nrounds = 1000,\n  early_stopping_rounds = 20,\n  verbose = 1\n)\n\n# Predict on validation set\npred_lgb <- predict(lgb_model, valid_matrix)\npred_lgb <- expm1(pmax(0, pred_lgb))  # Reverse log transformation and clip negative predictions\n\n# Calculate RMSLE\nrmsle <- sqrt(mean((log1p(pred_lgb) - log1p(expm1(valid_label)))^2))\ncat(\"RMSLE:\", round(rmsle, 4), \"\\n\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T10:43:41.389862Z","iopub.execute_input":"2024-12-28T10:43:41.39159Z","iopub.status.idle":"2024-12-28T10:43:57.66643Z","shell.execute_reply":"2024-12-28T10:43:57.664381Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Compare True vs. Predicted distributions\nvalid_actual <- expm1(valid_label)  # Reverse log transformation\nvalid_predictions <- pred_lgb\n\n# Create a combined dataframe for visualization\ncomparison_df <- data.frame(\n  Value = c(valid_actual, valid_predictions),\n  Type = rep(c(\"True\", \"Predicted\"), each = length(valid_actual))\n)\n\nggplot() +\n  geom_histogram(aes(x = valid_actual, y = after_stat(density), fill = \"True\"), \n                 bins = 30, alpha = 0.5, color = \"black\") +\n  geom_histogram(aes(x = valid_predictions, y = after_stat(density), fill = \"Predicted\"), \n                 bins = 30, alpha = 0.5, color = \"black\") +\n  geom_density(aes(x = valid_actual, color = \"True\"), linewidth = 1, linetype = \"dashed\") +\n  geom_density(aes(x = valid_predictions, color = \"Predicted\"), linewidth = 1) +\n  scale_x_log10() +  # Logarithmic scale for better visualization\n  labs(\n    title = \"Comparison of True and Predicted Distributions (Log Scale)\",\n    x = \"Log of Premium Amount\",\n    y = \"Density\",\n    fill = \"Type\",\n    color = \"Density\"\n  ) +\n  theme_minimal() +\n  theme(legend.position = \"top\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T10:44:14.369774Z","iopub.execute_input":"2024-12-28T10:44:14.371379Z","iopub.status.idle":"2024-12-28T10:44:17.567946Z","shell.execute_reply":"2024-12-28T10:44:17.565833Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Display Feature Importance\nlgb.importance(lgb_model) %>%\n  lgb.plot.importance()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T10:44:24.525635Z","iopub.execute_input":"2024-12-28T10:44:24.527381Z","iopub.status.idle":"2024-12-28T10:44:29.561774Z","shell.execute_reply":"2024-12-28T10:44:29.559797Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Predict on test set and submission","metadata":{}},{"cell_type":"code","source":"# Predict on the test dataset (unseen data)\nfinal_test_matrix <- as.matrix(test %>% select(-all_of(c(\"id\", \"policy_start_date\"))))\nfinal_test_predictions <- expm1(pmax(0, predict(lgb_model, final_test_matrix)))\n\n# Create submission file\nsubmission <- test %>%\n  select(id) %>%\n  mutate(`Premium Amount` = final_test_predictions)\n\nwrite.csv(submission, \"submission.csv\", row.names = FALSE)\ncat(\"Submission file saved as 'submission.csv'.\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T10:44:36.833518Z","iopub.execute_input":"2024-12-28T10:44:36.835163Z","iopub.status.idle":"2024-12-28T10:44:50.531124Z","shell.execute_reply":"2024-12-28T10:44:50.529279Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Display the first few rows of test with predictions\nhead(submission)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T10:44:55.172271Z","iopub.execute_input":"2024-12-28T10:44:55.174884Z","iopub.status.idle":"2024-12-28T10:44:55.213846Z","shell.execute_reply":"2024-12-28T10:44:55.212011Z"}},"outputs":[],"execution_count":null}]}