{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n#Import libraries\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport scipy.stats as sts\nfrom scipy.stats import skew\nfrom scipy.stats import kurtosis\n\nimport statistics\nimport warnings","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:29:52.747912Z","iopub.execute_input":"2024-12-31T21:29:52.748327Z","iopub.status.idle":"2024-12-31T21:29:52.760496Z","shell.execute_reply.started":"2024-12-31T21:29:52.748298Z","shell.execute_reply":"2024-12-31T21:29:52.758873Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#reading csv files\ndf = pd.read_csv(\"/kaggle/input/playground-series-s4e12/sample_submission.csv\")\ntrain = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:29:52.762524Z","iopub.execute_input":"2024-12-31T21:29:52.762958Z","iopub.status.idle":"2024-12-31T21:30:02.31647Z","shell.execute_reply.started":"2024-12-31T21:29:52.762912Z","shell.execute_reply":"2024-12-31T21:30:02.315239Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#concating the dataframes by removing the duplicates \ndf_combined = pd.concat([df.reset_index(drop=True),test.reset_index(drop = True),train.reset_index(drop=True)],axis=1)\ndf_combined = df_combined.loc[:,~df_combined.columns.duplicated()]\ndf_combined.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:30:02.318841Z","iopub.execute_input":"2024-12-31T21:30:02.319181Z","iopub.status.idle":"2024-12-31T21:30:03.612708Z","shell.execute_reply.started":"2024-12-31T21:30:02.319154Z","shell.execute_reply":"2024-12-31T21:30:03.611508Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_combined.columns = df_combined.columns.str.replace(' ', '')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:30:03.6145Z","iopub.execute_input":"2024-12-31T21:30:03.61493Z","iopub.status.idle":"2024-12-31T21:30:03.621424Z","shell.execute_reply.started":"2024-12-31T21:30:03.614898Z","shell.execute_reply":"2024-12-31T21:30:03.61992Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Counting null values\ndf_combined.isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:30:03.622921Z","iopub.execute_input":"2024-12-31T21:30:03.623317Z","iopub.status.idle":"2024-12-31T21:30:04.214753Z","shell.execute_reply.started":"2024-12-31T21:30:03.623276Z","shell.execute_reply":"2024-12-31T21:30:04.213528Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#viewing the datatypes\ndf_combined.dtypes","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:30:04.215999Z","iopub.execute_input":"2024-12-31T21:30:04.216336Z","iopub.status.idle":"2024-12-31T21:30:04.227962Z","shell.execute_reply.started":"2024-12-31T21:30:04.216306Z","shell.execute_reply":"2024-12-31T21:30:04.226569Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#differentiating the categorical and numerical in a datasets\nnumerical_columns = df_combined[['id','PremiumAmount','Age','AnnualIncome','NumberofDependents','HealthScore','PreviousClaims','VehicleAge','CreditScore','InsuranceDuration']]\ncategorical_columns = df_combined[['Gender','MaritalStatus','EducationLevel','Occupation','Location','PolicyType','CustomerFeedback','SmokingStatus','ExerciseFrequency','PropertyType','PolicyStartDate']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:35:35.36804Z","iopub.execute_input":"2024-12-31T21:35:35.368473Z","iopub.status.idle":"2024-12-31T21:35:35.569007Z","shell.execute_reply.started":"2024-12-31T21:35:35.368444Z","shell.execute_reply":"2024-12-31T21:35:35.567837Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#data imputation\nnumerical_columns= ['id','PremiumAmount','Age','AnnualIncome','NumberofDependents','HealthScore','PreviousClaims','VehicleAge','CreditScore','InsuranceDuration']\ndf_numeric = df_combined[numerical_columns]\n\n# Step 2: Apply np.isnan() to check for NaN values, then sum the NaNs per column\nmissing_values_count = df_numeric.apply(lambda col: np.isnan(col).sum())\n\n# Step 3: Calculate the percentage of missing values for each column\nmissing_percentage = (missing_values_count / len(df_combined)) * 100\n\n# Print the missing percentage\nprint(missing_percentage)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:36:11.842319Z","iopub.execute_input":"2024-12-31T21:36:11.842749Z","iopub.status.idle":"2024-12-31T21:36:11.936929Z","shell.execute_reply.started":"2024-12-31T21:36:11.842718Z","shell.execute_reply":"2024-12-31T21:36:11.935496Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#checking the percentage of null values\ncategorical_columns = ['Gender','MaritalStatus','EducationLevel','Occupation','Location','PolicyType','CustomerFeedback','SmokingStatus','ExerciseFrequency','PropertyType']\ndf_categoric = df_combined[categorical_columns]\n# Step 2: Apply np.isnan() to check for NaN values, then sum the NaNs per column\n# Calculate the percentage of each category in each column\ncategory_percentages = df_categoric.apply(lambda col: col.value_counts(normalize=True) * 100)\n# Step 3: Calculate the percentage of missing values for each column\nmissing_percentage_ca = (category_percentages/ len(df_combined)) * 100\nprint(missing_percentage_ca)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:36:42.693697Z","iopub.execute_input":"2024-12-31T21:36:42.694319Z","iopub.status.idle":"2024-12-31T21:36:43.637656Z","shell.execute_reply.started":"2024-12-31T21:36:42.694271Z","shell.execute_reply":"2024-12-31T21:36:43.636485Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_combined['PolicyStartDate'] = pd.to_datetime(df_combined['PolicyStartDate'], errors='coerce')\ndf_date = df_combined['PolicyStartDate'].dt.date\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:37:10.6821Z","iopub.execute_input":"2024-12-31T21:37:10.682543Z","iopub.status.idle":"2024-12-31T21:37:11.314505Z","shell.execute_reply.started":"2024-12-31T21:37:10.682513Z","shell.execute_reply":"2024-12-31T21:37:11.313223Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"numerical_columns = df_combined[['id','PremiumAmount','Age','AnnualIncome','NumberofDependents','HealthScore','PreviousClaims','VehicleAge','CreditScore','InsuranceDuration']].median()\ndf_combined[['id','PremiumAmount','Age','AnnualIncome','NumberofDependents','HealthScore','PreviousClaims','VehicleAge','CreditScore','InsuranceDuration']].fillna(numerical_columns)\ndf_combined.isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:39:31.745066Z","iopub.execute_input":"2024-12-31T21:39:31.745559Z","iopub.status.idle":"2024-12-31T21:39:32.769074Z","shell.execute_reply.started":"2024-12-31T21:39:31.745523Z","shell.execute_reply":"2024-12-31T21:39:32.768191Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"numerical_columns = df_combined[['id','PremiumAmount','Age','AnnualIncome','NumberofDependents','HealthScore','PreviousClaims','VehicleAge','CreditScore','InsuranceDuration']].median().iloc[0]\ndf_combined[['id','PremiumAmount','Age','AnnualIncome','NumberofDependents','HealthScore','PreviousClaims','VehicleAge','CreditScore','InsuranceDuration']].fillna(numerical_columns)\ndf_combined","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:42:06.864856Z","iopub.execute_input":"2024-12-31T21:42:06.865238Z","iopub.status.idle":"2024-12-31T21:42:08.679717Z","shell.execute_reply.started":"2024-12-31T21:42:06.865211Z","shell.execute_reply":"2024-12-31T21:42:08.678443Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mean_value = df_combined['PremiumAmount'].mean()\nmed_value = df_combined['PremiumAmount'].median()\nmode_value = df_combined['PremiumAmount'].mode()\nskewness = round(sts.skew(df_combined['PremiumAmount'],axis=0,bias=True),2)\nkurtosis = round(sts.kurtosis(df_combined['PremiumAmount'],axis=0,bias=True,fisher=0),2)\niqr = round(sts.iqr(df_combined['PremiumAmount'],axis=0,rng=(25,75)),2)\n\nprint('Mean:',mean_value)\nprint('Median:',med_value)\nprint('Mode:',mode_value)\nprint('skewness:',skewness)   \nprint('kurtosis:',kurtosis)\nprint('iqr:',iqr)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:43:03.926901Z","iopub.execute_input":"2024-12-31T21:43:03.927314Z","iopub.status.idle":"2024-12-31T21:43:03.996977Z","shell.execute_reply.started":"2024-12-31T21:43:03.927284Z","shell.execute_reply":"2024-12-31T21:43:03.994811Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mean_value = df_combined['Age'].mean()\nmedian_value = df_combined['Age'].median()\nmode_value = df_combined['Age'].mode()\nskewness = round(sts.skew(df_combined['Age'],axis=0,bias=True),2)\nkurtosis = round(sts.kurtosis(df_combined['Age'],axis=0,bias=True,fisher=0),2)\niqr = (sts.iqr(df_combined['Age'],axis=0,rng=(25,75)),2)\n\n\nprint(\"Mean:\",mean_value)\nprint(\"Median:\",median_value)\nprint(\"mode_value\",mode_value)\nprint(\"skewness\",skewness)\nprint(\"kurtosis\",kurtosis)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:30:04.281687Z","iopub.status.idle":"2024-12-31T21:30:04.282142Z","shell.execute_reply":"2024-12-31T21:30:04.281948Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mean_value = df_combined['AnnualIncome'].mean()\nmedian_value = df_combined['AnnualIncome'].median()\nmode_value = df_combined['AnnualIncome'].mode()\nskewness = round(sts.skew(df_combined['AnnualIncome'],axis=0,bias=True),2)\nkurtosis = round(sts.kurtosis(df_combined['AnnualIncome'],axis=0,bias=True, fisher=0),2)\niqr = (sts.iqr(df_combined['AnnualIncome'],axis=0,rng=(25,75)),2)\n\n\nprint(\"Mean:\",mean_value)\nprint(\"Median:\",median_value)\nprint(\"mode_value\",mode_value)\nprint(\"skewness\",skewness)\nprint(\"kurtosis\",kurtosis)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:43:24.132778Z","iopub.execute_input":"2024-12-31T21:43:24.133235Z","iopub.status.idle":"2024-12-31T21:43:24.252461Z","shell.execute_reply.started":"2024-12-31T21:43:24.133203Z","shell.execute_reply":"2024-12-31T21:43:24.251313Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mean_value = df_combined['NumberofDependents'].mean()\nmedian_value = df_combined['NumberofDependents'].median()\nmode_value = df_combined['NumberofDependents'].mode()\nskewness = round(sts.skew(df_combined['NumberofDependents'],axis=0,bias=True),2)\nkurtosis = round(sts.kurtosis(df_combined['NumberofDependents'],axis=0,bias=True, fisher=0),2)\niqr = (sts.iqr(df_combined['NumberofDependents'],axis=0,rng=(25,75)),2)\n\n\nprint(\"Mean:\",mean_value)\nprint(\"Median:\",median_value)\nprint(\"mode_value\",mode_value)\nprint(\"skewness\",skewness)\nprint(\"kurtosis\",kurtosis)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:43:55.444882Z","iopub.execute_input":"2024-12-31T21:43:55.445265Z","iopub.status.idle":"2024-12-31T21:43:55.5281Z","shell.execute_reply.started":"2024-12-31T21:43:55.445236Z","shell.execute_reply":"2024-12-31T21:43:55.526517Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mean_value = df_combined['HealthScore'].mean()\nmedian_value = df_combined['HealthScore'].median()\nmode_value = df_combined['HealthScore'].mode()\nskewness = round(sts.skew(df_combined['HealthScore'],axis=0,bias=True),2)\nkurtosis = round(sts.kurtosis(df_combined['HealthScore'],axis=0,bias=True, fisher=0),2)\niqr = (sts.iqr(df_combined['HealthScore'],axis=0,rng=(25,75)),2)\n\n\nprint(\"Mean:\",mean_value)\nprint(\"Median:\",median_value)\nprint(\"mode_value\",mode_value)\nprint(\"skewness\",skewness)\nprint(\"kurtosis\",kurtosis)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:44:14.859912Z","iopub.execute_input":"2024-12-31T21:44:14.860327Z","iopub.status.idle":"2024-12-31T21:44:15.028805Z","shell.execute_reply.started":"2024-12-31T21:44:14.8603Z","shell.execute_reply":"2024-12-31T21:44:15.027318Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mean_value = df_combined['PreviousClaims'].mean()\nmedian_value = df_combined['PreviousClaims'].median()\nmode_value = df_combined['PreviousClaims'].mode()\nskewness = round(sts.skew(df_combined['PreviousClaims'],axis=0,bias=True),2)\nkurtosis = round(sts.kurtosis(df_combined['PreviousClaims'],axis=0,bias=True, fisher=0),2)\niqr = (sts.iqr(df_combined['PreviousClaims'],axis=0,rng=(25,75)),2)\n\n\nprint(\"Mean:\",mean_value)\nprint(\"Median:\",median_value)\nprint(\"mode_value\",mode_value)\nprint(\"skewness\",skewness)\nprint(\"kurtosis\",kurtosis)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:44:41.512789Z","iopub.execute_input":"2024-12-31T21:44:41.513137Z","iopub.status.idle":"2024-12-31T21:44:41.600535Z","shell.execute_reply.started":"2024-12-31T21:44:41.513112Z","shell.execute_reply":"2024-12-31T21:44:41.599156Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mean_value = df_combined['VehicleAge'].mean()\nmedian_value = df_combined['VehicleAge'].median()\nmode_value = df_combined['VehicleAge'].mode()\nskewness = round(sts.skew(df_combined['VehicleAge'],axis=0,bias=True),2)\nkurtosis = round(sts.kurtosis(df_combined['VehicleAge'],axis=0,bias=True, fisher=0),2)\niqr = (sts.iqr(df_combined['VehicleAge'],axis=0,rng=(25,75)),2)\n\n\nprint(\"Mean:\",mean_value)\nprint(\"Median:\",median_value)\nprint(\"mode_value\",mode_value)\nprint(\"skewness\",skewness)\nprint(\"kurtosis\",kurtosis)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:45:02.773459Z","iopub.execute_input":"2024-12-31T21:45:02.773907Z","iopub.status.idle":"2024-12-31T21:45:02.843714Z","shell.execute_reply.started":"2024-12-31T21:45:02.773876Z","shell.execute_reply":"2024-12-31T21:45:02.842319Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mean_value = df_combined['CreditScore'].mean()\nmedian_value = df_combined['CreditScore'].median()\nmode_value = df_combined['CreditScore'].mode()\nskewness = round(sts.skew(df_combined['CreditScore'],axis=0,bias=True),2)\nkurtosis = round(sts.kurtosis(df_combined['CreditScore'],axis=0,bias=True, fisher=0),2)\niqr = (sts.iqr(df_combined['CreditScore'],axis=0,rng=(25,75)),2)\n\n\nprint(\"Mean:\",mean_value)\nprint(\"Median:\",median_value)\nprint(\"mode_value\",mode_value)\nprint(\"skewness\",skewness)\nprint(\"kurtosis\",kurtosis)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:45:20.986142Z","iopub.execute_input":"2024-12-31T21:45:20.986543Z","iopub.status.idle":"2024-12-31T21:45:21.064418Z","shell.execute_reply.started":"2024-12-31T21:45:20.986516Z","shell.execute_reply":"2024-12-31T21:45:21.063179Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"median_value = df_combined['InsuranceDuration'].median()\nmode_value = df_combined['InsuranceDuration'].mode()\nskewness = round(sts.skew(df_combined['InsuranceDuration'],axis=0,bias=True),2)\nkurtosis = round(sts.kurtosis(df_combined['InsuranceDuration'],axis=0,bias=True, fisher=0),2)\niqr = (sts.iqr(df_combined['InsuranceDuration'],axis=0,rng=(25,75)),2)\n\n\nprint(\"Mean:\",mean_value)\nprint(\"Median:\",median_value)\nprint(\"mode_value\",mode_value)\nprint(\"skewness\",skewness)\nprint(\"kurtosis\",kurtosis)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:45:35.635678Z","iopub.execute_input":"2024-12-31T21:45:35.63613Z","iopub.status.idle":"2024-12-31T21:45:35.714027Z","shell.execute_reply.started":"2024-12-31T21:45:35.636097Z","shell.execute_reply":"2024-12-31T21:45:35.711721Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_combined","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:30:04.296309Z","iopub.status.idle":"2024-12-31T21:30:04.296999Z","shell.execute_reply":"2024-12-31T21:30:04.296736Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calculate the max value of 'Insurance Duration', excluding np.inf values\nmax_infinite_values = df_combined[~np.isinf(df_combined['InsuranceDuration'])]['InsuranceDuration'].max()\nmax_infinite_valuess = df_combined[~np.isinf(df_combined['PremiumAmount'])]['PremiumAmount'].max()\nmax_infinite_valuesss = df_combined[~np.isinf(df_combined['PreviousClaims'])]['PreviousClaims'].max()\nmax_infinite = df_combined[~np.isinf(df_combined['CreditScore'])]['CreditScore'].max()\n\n# Replace np.inf values with the calculated max value\ndf_combined['InsuranceDuration'] = df_combined['InsuranceDuration'].replace(np.inf,max_infinite_values)\ndf_combined['PremiumAmount']= df_combined['PremiumAmount'].replace(np.inf,max_infinite_valuess)\n\n# Print the max value (used to replace np.inf)\nprint(\"Max value for replacement of np.inf:\", max_infinite_values)\nprint(\"max value for Premium Amount\",max_infinite_valuess)\nprint(\"max value for Previous Claims:\",max_infinite_valuesss)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:46:17.395735Z","iopub.execute_input":"2024-12-31T21:46:17.396193Z","iopub.status.idle":"2024-12-31T21:46:20.620405Z","shell.execute_reply.started":"2024-12-31T21:46:17.396162Z","shell.execute_reply":"2024-12-31T21:46:20.61925Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#max_infinite_values = df_combined[~np.isinf(df_combined['Insurance Duration'])].max()\ndf_combined['InsuranceDuration']\ndf_combined['InsuranceDuration'].isna().sum()\ndrop_duplicates = df_combined['InsuranceDuration'].dropna()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:30:04.299695Z","iopub.status.idle":"2024-12-31T21:30:04.300246Z","shell.execute_reply":"2024-12-31T21:30:04.300025Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#checking the percentage of null values\ncategorical_columns = ['Gender','MaritalStatus','EducationLevel','Occupation','Location','PolicyType','CustomerFeedback','SmokingStatus','ExerciseFrequency','PropertyType']\ndf_categoric = df_combined[categorical_columns]\n\n# Step 2: Apply np.isnan() to check for NaN values, then sum the NaNs per column\n# Calculate the percentage of each category in each column\ncategory_percentages = df_categoric.apply(lambda col: col.value_counts(normalize=True) * 100)\n# Step 3: Calculate the percentage of missing values for each column\nmissing_percentage_ca = (category_percentages/ len(df_combined)) * 100\nprint(missing_percentage_ca)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:47:08.1247Z","iopub.execute_input":"2024-12-31T21:47:08.125137Z","iopub.status.idle":"2024-12-31T21:47:08.978288Z","shell.execute_reply.started":"2024-12-31T21:47:08.125105Z","shell.execute_reply":"2024-12-31T21:47:08.976752Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#mode imputation for categorical columns\ncategorical_columns = df_combined[['Gender','MaritalStatus','EducationLevel','Occupation','Location','PolicyType','CustomerFeedback','SmokingStatus','ExerciseFrequency','PropertyType']].mode().iloc[0]\ndf_combined = df_combined[['Gender','MaritalStatus','EducationLevel','Occupation','Location','PolicyType','CustomerFeedback','SmokingStatus','ExerciseFrequency','PropertyType']].fillna(categorical_columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:51:22.960905Z","iopub.execute_input":"2024-12-31T21:51:22.961372Z","iopub.status.idle":"2024-12-31T21:51:25.394867Z","shell.execute_reply.started":"2024-12-31T21:51:22.961323Z","shell.execute_reply":"2024-12-31T21:51:25.393723Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_combined","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:51:32.686795Z","iopub.execute_input":"2024-12-31T21:51:32.687188Z","iopub.status.idle":"2024-12-31T21:51:32.704522Z","shell.execute_reply.started":"2024-12-31T21:51:32.687161Z","shell.execute_reply":"2024-12-31T21:51:32.70313Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"numerical_columns= ['id','PremiumAmount','Age','AnnualIncome','NumberofDependents','HealthScore','PreviousClaims','VehicleAge','CreditScore','Insurance Duration']\ncategorical_columns = ['Gender','MaritalStatus','EducationLevel','Occupation','Location','PolicyType','CustomerFeedback','SmokingStatus','Exercise Frequency','Property Type']\nmax_len = max(len(numerical_columns),len(categorical_columns))\nprint(max_len)\ncategorical_columns.extend([None] * (max_len - len(categorical_columns)))\nprint(categorical_columns)\nmissing_numerical_columns = [col for col in numerical_columns if col not in df_combined.columns]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:52:20.013918Z","iopub.execute_input":"2024-12-31T21:52:20.0143Z","iopub.status.idle":"2024-12-31T21:52:20.021937Z","shell.execute_reply.started":"2024-12-31T21:52:20.014272Z","shell.execute_reply":"2024-12-31T21:52:20.020635Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":" # Check if all numerical columns exist in the dataframe\nmissing_numerical_cols = [col for col in numerical_columns if col not in df_combined.columns]\n\n# If there are missing numerical columns, print them\nif missing_numerical_cols:\n    print(f\"Missing numerical columns: {missing_numerical_cols}\")\nelse:\n    # Pad categorical columns if necessary\n    max_len = max(len(numerical_columns), len(categorical_columns))\n    categorical_columns.extend([None] * (max_len - len(categorical_columns)))  # Add None to categorical columns to match max length\n    \n    # Select numerical columns from df_combined\n    df_numeric = df_combined[numerical_columns]\n    print(\"df_numeric:\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:52:27.680644Z","iopub.execute_input":"2024-12-31T21:52:27.681019Z","iopub.status.idle":"2024-12-31T21:52:27.687937Z","shell.execute_reply.started":"2024-12-31T21:52:27.680994Z","shell.execute_reply":"2024-12-31T21:52:27.685974Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"columns_to_include = [numerical_columns, categorical_columns]\n# Check if the lists have the same length\nif len(numerical_columns) == len(categorical_columns):\n    # Convert lists to pandas Series\n    numerical_series = pd.Series(numerical_columns)\n    categorical_series = pd.Series(categorical_columns)\n    \n    # Create DataFrame using the series\n    frame = {'numerical_columns': numerical_series, 'categorical_columns': categorical_series}\n    new_df = pd.DataFrame(frame)\n    \n    # Print the new DataFrame\n    print(new_df)\nelse:\n    # If lists have different lengths, print an error\n    print(\"Error: The lists must have the same length.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:52:44.071208Z","iopub.execute_input":"2024-12-31T21:52:44.071601Z","iopub.status.idle":"2024-12-31T21:52:44.082419Z","shell.execute_reply.started":"2024-12-31T21:52:44.071572Z","shell.execute_reply":"2024-12-31T21:52:44.080247Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"new_df = pd.DataFrame(frame)\nnew_df","metadata":{"execution":{"iopub.status.busy":"2024-12-29T19:14:15.002543Z","iopub.execute_input":"2024-12-29T19:14:15.002919Z","iopub.status.idle":"2024-12-29T19:14:15.007522Z","shell.execute_reply.started":"2024-12-29T19:14:15.00289Z","shell.execute_reply":"2024-12-29T19:14:15.006377Z"}}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{"execution":{"iopub.status.busy":"2024-12-29T19:14:15.002543Z","iopub.execute_input":"2024-12-29T19:14:15.002919Z","iopub.status.idle":"2024-12-29T19:14:15.007522Z","shell.execute_reply.started":"2024-12-29T19:14:15.00289Z","shell.execute_reply":"2024-12-29T19:14:15.006377Z"}}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"  ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}