{"metadata":{"kernelspec":{"name":"ir","display_name":"R","language":"R"},"language_info":{"name":"R","codemirror_mode":"r","pygments_lexer":"r","mimetype":"text/x-r-source","file_extension":".r","version":"4.4.0"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30749,"isInternetEnabled":true,"language":"r","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction   \n\nIn this competion, the objectives  is to predict insurance premiums (Premium Amount) based on various factors.","metadata":{}},{"cell_type":"markdown","source":"# Clear workspace and load data","metadata":{"execution":{"iopub.status.busy":"2024-12-26T19:11:15.534044Z","iopub.execute_input":"2024-12-26T19:11:15.535979Z","iopub.status.idle":"2024-12-26T19:11:15.555233Z","shell.execute_reply":"2024-12-26T19:11:15.553282Z"}}},{"cell_type":"code","source":"rm(list=ls())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T12:26:48.035216Z","iopub.execute_input":"2025-01-04T12:26:48.03708Z","iopub.status.idle":"2025-01-04T12:26:48.147937Z","shell.execute_reply":"2025-01-04T12:26:48.14622Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# libraries\nsuppressMessages({library(tidyverse) \n                  library(ggcorrplot)\n                  library(mice)\n                  library(missForest)\n                  library(parallel)\n                  library(doParallel)\n                  }\n                 )","metadata":{"_uuid":"051d70d956493feee0c6d64651c6a088724dca2a","_execution_state":"idle","trusted":true,"execution":{"iopub.status.busy":"2025-01-04T12:26:48.150531Z","iopub.execute_input":"2025-01-04T12:26:48.185458Z","iopub.status.idle":"2025-01-04T12:26:51.304854Z","shell.execute_reply":"2025-01-04T12:26:51.292925Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"path <- \"../input/playground-series-s4e12/\"\nlist.files(path = path)","metadata":{"_uuid":"051d70d956493feee0c6d64651c6a088724dca2a","_execution_state":"idle","trusted":true,"execution":{"iopub.status.busy":"2025-01-04T12:26:51.307719Z","iopub.execute_input":"2025-01-04T12:26:51.309509Z","iopub.status.idle":"2025-01-04T12:26:51.331597Z","shell.execute_reply":"2025-01-04T12:26:51.329967Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# load data\ntrain <- read_csv(paste0(path,\"train.csv\"),show_col_types = FALSE)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T12:26:51.334062Z","iopub.execute_input":"2025-01-04T12:26:51.335429Z","iopub.status.idle":"2025-01-04T12:26:57.642731Z","shell.execute_reply":"2025-01-04T12:26:57.640867Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data analysis","metadata":{}},{"cell_type":"markdown","source":"## Data wrangling","metadata":{}},{"cell_type":"code","source":"#data structure\n#str(train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T13:11:51.97787Z","iopub.execute_input":"2025-01-04T13:11:51.979412Z","iopub.status.idle":"2025-01-04T13:11:51.989264Z","shell.execute_reply":"2025-01-04T13:11:51.987669Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"skimr::skim(train)-> trainSummary","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T12:26:57.701548Z","iopub.execute_input":"2025-01-04T12:26:57.702898Z","iopub.status.idle":"2025-01-04T12:27:06.88963Z","shell.execute_reply":"2025-01-04T12:27:06.887968Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# na_proportion >= 0.05\ndata.frame(na_prop = colMeans(is.na(train))) |>\nround(2)|> \nfilter(na_prop >=0.05)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T12:27:06.89371Z","iopub.execute_input":"2025-01-04T12:27:06.895254Z","iopub.status.idle":"2025-01-04T12:27:07.336837Z","shell.execute_reply":"2025-01-04T12:27:07.334596Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## features with na_proportion between 0 and  0.05\ndata.frame(na_prop = colMeans(is.na(train))) |>\nfilter(0 < na_prop & na_prop <=0.05)|>\nround(3) ","metadata":{"execution":{"iopub.status.busy":"2025-01-04T12:27:07.629794Z","iopub.execute_input":"2025-01-04T12:27:07.631112Z","iopub.status.idle":"2025-01-04T12:27:07.893188Z","shell.execute_reply":"2025-01-04T12:27:07.890808Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# features with less than 5% NA  \n(trainSummary$skim_variable[0 < (1 - trainSummary$complete_rate) & (1 - trainSummary$complete_rate) <=0.05]-> na_features)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T12:27:39.860494Z","iopub.execute_input":"2025-01-04T12:27:39.861957Z","iopub.status.idle":"2025-01-04T12:27:39.879186Z","shell.execute_reply":"2025-01-04T12:27:39.877008Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# remove id \ntrain$id <- NULL","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T12:27:44.50168Z","iopub.execute_input":"2025-01-04T12:27:44.503113Z","iopub.status.idle":"2025-01-04T12:27:44.516866Z","shell.execute_reply":"2025-01-04T12:27:44.515256Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rename_with(train, ~ gsub(\" \", \"_\", .x, fixed = TRUE))-> train\ndim(train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T12:27:46.800571Z","iopub.execute_input":"2025-01-04T12:27:46.802024Z","iopub.status.idle":"2025-01-04T12:27:46.819154Z","shell.execute_reply":"2025-01-04T12:27:46.817615Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Imputing training data","metadata":{}},{"cell_type":"code","source":"# imputing numeric features\na <- Sys.time()\ntrain |>\nselect_if(is.numeric)|>\nmice(m=5,seed=10,print=F)|> \ncomplete()-> data2\nb <- Sys.time()\n\n# elapsed time \ncat(\"\\n\")\ndifftime(b,a,units = \"mins\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T12:28:23.583779Z","iopub.execute_input":"2025-01-04T12:28:23.585314Z","iopub.status.idle":"2025-01-04T12:38:10.612561Z","shell.execute_reply":"2025-01-04T12:38:10.61061Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data.frame(prop_na = colMeans(is.na(data2)))|> \nfilter(prop_na > 0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T12:38:10.616367Z","iopub.execute_input":"2025-01-04T12:38:10.617675Z","iopub.status.idle":"2025-01-04T12:38:10.676609Z","shell.execute_reply":"2025-01-04T12:38:10.674945Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dim(data2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T12:38:10.679088Z","iopub.execute_input":"2025-01-04T12:38:10.680391Z","iopub.status.idle":"2025-01-04T12:38:10.693712Z","shell.execute_reply":"2025-01-04T12:38:10.692184Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data3 <- cbind(data2,train[setdiff(names(train),(names(data2)))])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T12:38:10.696183Z","iopub.execute_input":"2025-01-04T12:38:10.697434Z","iopub.status.idle":"2025-01-04T12:38:10.708131Z","shell.execute_reply":"2025-01-04T12:38:10.706695Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dim(data3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T13:11:18.296051Z","iopub.execute_input":"2025-01-04T13:11:18.297729Z","iopub.status.idle":"2025-01-04T13:11:18.31475Z","shell.execute_reply":"2025-01-04T13:11:18.312673Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# NA pproportion\nmean(is.na(data3))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T12:38:10.71056Z","iopub.execute_input":"2025-01-04T12:38:10.71175Z","iopub.status.idle":"2025-01-04T12:38:10.806195Z","shell.execute_reply":"2025-01-04T12:38:10.804648Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Only 2% of the entire dataset present NA values, we can get rid of these lines without a great information loss.","metadata":{}},{"cell_type":"code","source":"data.frame(prop_na = colMeans(is.na(data3)))|>\nfilter(prop_na > 0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T20:13:58.763951Z","iopub.execute_input":"2024-12-31T20:13:58.765745Z","iopub.status.idle":"2024-12-31T20:13:58.964172Z","shell.execute_reply":"2024-12-31T20:13:58.961946Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Features selection with lm : which features do influence Premium_Amount # ?","metadata":{}},{"cell_type":"code","source":"lm(Premium_Amount~.-1, data3) |>\ntidy()|>\ndata.frame()|>\nfilter(p.value < 0.05)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T13:08:55.157888Z","iopub.execute_input":"2025-01-04T13:08:55.159597Z","iopub.status.idle":"2025-01-04T13:08:57.735726Z","shell.execute_reply":"2025-01-04T13:08:57.731921Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat(round((mean(complete.cases(data3))),2)*1e2,\"% of original dataset is NA free\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T13:07:49.668589Z","iopub.execute_input":"2025-01-04T13:07:49.670084Z","iopub.status.idle":"2025-01-04T13:07:49.781208Z","shell.execute_reply":"2025-01-04T13:07:49.779473Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data4 <- data3[c(\"Age\",\"Annual_Income\",\"Gender\",\"Credit_Score\",\"Health_Score\",\n                \"Previous_Claims\",\"Policy_Start_Date\",\"Customer_Feedback\",\n                \"Premium_Amount\")]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T13:09:48.255707Z","iopub.execute_input":"2025-01-04T13:09:48.257282Z","iopub.status.idle":"2025-01-04T13:09:48.270049Z","shell.execute_reply":"2025-01-04T13:09:48.268508Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dim(data4)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T13:09:52.700795Z","iopub.execute_input":"2025-01-04T13:09:52.702389Z","iopub.status.idle":"2025-01-04T13:09:52.716517Z","shell.execute_reply":"2025-01-04T13:09:52.714879Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Turn character features into factors\nfor (col in names(data4)){\n  if(is.character(data4[[col]])) {\n      print(col)\n      data4[col]<-factor(data4[[col]])\n  } \n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T13:10:06.902764Z","iopub.execute_input":"2025-01-04T13:10:06.904354Z","iopub.status.idle":"2025-01-04T13:10:07.00957Z","shell.execute_reply":"2025-01-04T13:10:07.007917Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## features visualization","metadata":{}},{"cell_type":"code","source":"summary(data4$Age)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T13:14:57.10671Z","iopub.execute_input":"2025-01-04T13:14:57.108384Z","iopub.status.idle":"2025-01-04T13:14:57.178994Z","shell.execute_reply":"2025-01-04T13:14:57.176916Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"boxplot(data4$Age,\n       horizontal = T,\n       xlab = \"Age\") # Age without Nas\nabline(v = mean(data4$Age),col =\"red\")\nabline(v = median(data4$Age),col =\"blue\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T13:18:24.029414Z","iopub.execute_input":"2025-01-04T13:18:24.031006Z","iopub.status.idle":"2025-01-04T13:18:24.223503Z","shell.execute_reply":"2025-01-04T13:18:24.221678Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"summary(data4$Premium_Amount)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T13:20:43.06094Z","iopub.execute_input":"2025-01-04T13:20:43.062566Z","iopub.status.idle":"2025-01-04T13:20:43.135905Z","shell.execute_reply":"2025-01-04T13:20:43.134004Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#hist of premium Amount\nhist(data4$Premium_Amount,xlab = \"Premium Amount\",freq=F,main =\"distribution of Premium Amount  is right skewed\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T13:29:03.079594Z","iopub.execute_input":"2025-01-04T13:29:03.081134Z","iopub.status.idle":"2025-01-04T13:29:03.205602Z","shell.execute_reply":"2025-01-04T13:29:03.20291Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#hist of premium Amount\nhist(data4$Annual_Income,xlab = \"Annual_Income\",freq=F,main =\"distribution of Annual Income is right skewed\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T13:28:27.600125Z","iopub.execute_input":"2025-01-04T13:28:27.601573Z","iopub.status.idle":"2025-01-04T13:28:27.726682Z","shell.execute_reply":"2025-01-04T13:28:27.723886Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Is there any feature Correlation ?","metadata":{}},{"cell_type":"code","source":"# correlation between features\nlibrary(ggcorrplot)\ndata4 |>\n    select_if(is.numeric) |>\n    cor() |>\n    ggcorrplot()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T13:22:56.35946Z","iopub.execute_input":"2025-01-04T13:22:56.361019Z","iopub.status.idle":"2025-01-04T13:22:56.800758Z","shell.execute_reply":"2025-01-04T13:22:56.798986Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cor(data4$Credit_Score,data4$Annual_Income)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T13:23:31.876764Z","iopub.execute_input":"2025-01-04T13:23:31.878327Z","iopub.status.idle":"2025-01-04T13:23:31.906252Z","shell.execute_reply":"2025-01-04T13:23:31.90464Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"There is only a week correlation between Credit_Score and Annual_Income","metadata":{}},{"cell_type":"markdown","source":"TO BE CONTINUED 😉 ","metadata":{}}]}