{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"},{"sourceId":9178166,"sourceType":"datasetVersion","datasetId":5547076}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# - Description\n","metadata":{}},{"cell_type":"markdown","source":"![The-role-of-data-analytics-in-insurance.png](https://innowise.com/wp-content/uploads/2023/12/The-role-of-data-analytics-in-insurance.png)","metadata":{}},{"cell_type":"markdown","source":"The old discription: The goal of this dataset is to facilitate the development and testing of regression models for predicting insurance premiums based on various customer characteristics and policy details. Insurance companies often rely on data-driven approaches to estimate premiums, taking into account factors such as age, income, health status, and claim history. This synthetic dataset simulates real-world scenarios to help practitioners practice feature engineering, data cleaning, and model training.\nhttps://www.kaggle.com/code/gauravduttakiit/org-pss4e1","metadata":{}},{"cell_type":"markdown","source":"**Evaluation**\n\r\nSubmissions are evaluated using the Root Mean Squared Logarithmic Error (RMSLE).","metadata":{}},{"cell_type":"markdown","source":"![kaggle-forum-message-attachments/o/inbox%2F8418027%2Fcddd759e4dd26007634bcd01c2ea8a42%2FScreenshot%202024-01-10%20135652.png?generation=1704873450127348&alt=media](https://www.googleapis.com/download/storage/v1/b/kaggle-forum-message-attachments/o/inbox%2F8418027%2Fcddd759e4dd26007634bcd01c2ea8a42%2FScreenshot%202024-01-10%20135652.png?generation=1704873450127348&alt=media)","metadata":{}},{"cell_type":"markdown","source":"# - Welcome","metadata":{}},{"cell_type":"markdown","source":"Welcome to my github - https://github.com/Alexsandrrus/projekts-Kaggle","metadata":{}},{"cell_type":"markdown","source":"# - Libraries","metadata":{}},{"cell_type":"code","source":"import matplotlib\nimport matplotlib.pyplot as plt\nimport matplotlib.mlab as mlab\n\nimport re\nfrom scipy import stats\nimport statsmodels.api as sm\nfrom statsmodels.stats.outliers_influence import variance_inflation_factor\n\nimport pandas as pd\nimport seaborn as sns\nimport tensorflow as tf\nimport keras\nimport keras_tuner\nimport optuna\nfrom catboost import CatBoostRegressor\nfrom lightgbm import LGBMRegressor\nimport xgboost as xgb\n\nfrom keras import layers, regularizers\nfrom keras.layers import (BatchNormalization, Dense, Dropout, \n                           Flatten, Activation)\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.decomposition import PCA\nfrom sklearn.ensemble import (IsolationForest, \n                              VotingRegressor)\nfrom sklearn.linear_model import SGDOneClassSVM\nfrom sklearn.metrics import mean_squared_error, mean_squared_log_error,  mean_absolute_error\nfrom sklearn.model_selection import (KFold, StratifiedKFold, \n                                     StratifiedGroupKFold, \n                                     RepeatedStratifiedKFold, \n                                     RepeatedKFold, \n                                     train_test_split, \n                                     cross_validate)\nfrom sklearn.ensemble import (RandomForestRegressor, HistGradientBoostingRegressor,\n                                GradientBoostingRegressor, ExtraTreesRegressor)\nfrom sklearn.pipeline import Pipeline\nfrom sklearn import preprocessing\nfrom sklearn.preprocessing import (OneHotEncoder,LabelEncoder, MaxAbsScaler, \n                                   MinMaxScaler, PowerTransformer, \n                                   QuantileTransformer, \n                                   RobustScaler, StandardScaler)\nfrom sklearn.feature_selection import VarianceThreshold\n\nplt.style.use('ggplot')\n%matplotlib inline\nmatplotlib.rcParams['figure.figsize'] = (12, 8)\n\npd.options.mode.chained_assignment = None\n\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def plot_numerical_features(df):\n    num_features = df.select_dtypes(include=[np.number]).columns\n    ncols = 2\n    nrows = (len(num_features) + ncols - 1) // ncols\n\n    fig, axes = plt.subplots(nrows=nrows, ncols=ncols, figsize=(12, 5 * nrows))\n    axes = axes.flatten()\n\n    for i, feature in enumerate(num_features):\n        sns.histplot(df[feature], bins=30, kde=True, ax=axes[i])\n        axes[i].set_title(f'Distribution of {feature}')\n    \n    plt.tight_layout()\n    plt.show()\n\ndef plot_numerical_boxplots(df):\n    num_features = df.select_dtypes(include=[np.number]).columns\n    ncols = 2\n    nrows = (len(num_features) + ncols - 1) // ncols\n\n    fig, axes = plt.subplots(nrows=nrows, ncols=ncols, figsize=(12, 5 * nrows))\n    axes = axes.flatten()\n\n    for i, feature in enumerate(num_features):\n        sns.boxplot(x=df[feature], ax=axes[i])\n        axes[i].set_title(f'Boxplot of {feature}')\n    \n    plt.tight_layout()\n    plt.show()\n\ndef plot_qq_plot(df):\n    num_features = df.select_dtypes(include=[np.number]).columns\n    ncols = 2\n    nrows = (len(num_features) + ncols - 1) // ncols\n\n    fig, axes = plt.subplots(nrows=nrows, ncols=ncols, figsize=(12, 5 * nrows))\n    axes = axes.flatten()\n\n    for i, feature in enumerate(num_features):\n        stats.probplot(df[feature], dist=\"norm\", plot=axes[i])\n        axes[i].set_title(f'QQ Plot of {feature}')\n    \n    plt.tight_layout()\n    plt.show()\n\ndef plot_correlation_matrix(df, method='spearman'):\n    num_df = df.select_dtypes(include=[np.number])\n    \n    corr = num_df.corr(method=method)\n    plt.figure(figsize=(14, 10))\n    sns.heatmap(corr, annot=True, fmt=\".2f\", cmap='coolwarm', square=True, cbar_kws={\"shrink\": .8})\n    plt.title(f'Correlation Matrix ({method.capitalize()} Correlation)')\n    plt.show()\n\ndef plot_pairplot(df):\n    num_features = df.select_dtypes(include=[np.number]).columns\n    sns.pairplot(df[num_features])\n    plt.show()\n\ndef plot_categorical_features(df, ncols=2, top_n=None):\n    cat_features = df.select_dtypes(include=[object]).columns\n    nrows = (len(cat_features) + ncols - 1) // ncols\n\n    fig, axes = plt.subplots(nrows=nrows, ncols=ncols, figsize=(12, 5 * nrows))\n    axes = axes.flatten()\n\n    for i, feature in enumerate(cat_features):\n        if top_n is not None:\n            top_categories = df[feature].value_counts().nlargest(top_n).index\n            sns.countplot(data=df[df[feature].isin(top_categories)], y=feature, ax=axes[i])  \n        else:\n            sns.countplot(data=df, y=feature, ax=axes[i])  \n        \n        axes[i].set_title(f'Count of {feature}')\n        axes[i].tick_params(axis='y', rotation=0)  \n\n    for j in range(i + 1, len(axes)):\n        fig.delaxes(axes[j])\n\n    plt.tight_layout()\n    plt.show()\n\n    \ndef day_part(hour):\n    if hour in [4,5]:\n        return 1\n    elif hour in [6,7]:\n        return 2\n    elif hour in [8,9,10]:\n        return 3\n    elif hour in [11,12,13]:\n        return 4\n    elif hour in [14,15,16]:\n        return 5\n    elif hour in [17, 18,19]:\n        return 6\n    elif hour in [20, 21, 22]:\n        return 7\n    elif hour in [23,0,1,2,3]:\n        return 0\n\ndef PolynomialFeatures_labeled(input_df,power):\n   \n    poly = preprocessing.PolynomialFeatures(power)\n    output_nparray = poly.fit_transform(input_df)\n    powers_nparray = poly.powers_\n\n    input_feature_names = list(input_df.columns)\n    target_feature_names = [\"Constant Term\"]\n    for feature_distillation in powers_nparray[1:]:\n        intermediary_label = \"\"\n        final_label = \"\"\n        for i in range(len(input_feature_names)):\n            if feature_distillation[i] == 0:\n                continue\n            else:\n                variable = input_feature_names[i]\n                power = feature_distillation[i]\n                intermediary_label = \"%s+%d\" % (variable,power)\n                if final_label == \"\":         #If the final label isn't yet specified\n                    final_label = intermediary_label\n                else:\n                    final_label = final_label + \"x\" + intermediary_label\n        target_feature_names.append(final_label)\n    output_df = pd.DataFrame(output_nparray, columns = target_feature_names)\n    return output_df\n\ndef variance_threshold(df,th):\n    var_thres=VarianceThreshold(threshold=th)\n    var_thres.fit(df)\n    new_cols = var_thres.get_support()\n    return df.iloc[:,new_cols]\n   \ndef optimize_memory_usage(df, print_size=True):\n# Function optimizes memory usage in dataframe.\n\n# Types for optimization.\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    # Memory usage size before optimize (Mb).\n    before_size = df.memory_usage().sum() / 1024**2    \n    for column in df.columns:\n        column_type = df[column].dtypes\n        if column_type in numerics:\n            column_min = df[column].min()\n            column_max = df[column].max()\n            if str(column_type).startswith('int'):\n                if column_min > np.iinfo(np.int8).min and column_max < np.iinfo(np.int8).max:\n                    df[column] = df[column].astype(np.int8)\n                elif column_min > np.iinfo(np.int16).min and column_max < np.iinfo(np.int16).max:\n                    df[column] = df[column].astype(np.int16)\n                elif column_min > np.iinfo(np.int32).min and column_max < np.iinfo(np.int32).max:\n                    df[column] = df[column].astype(np.int32)\n                elif column_min > np.iinfo(np.int64).min and column_max < np.iinfo(np.int64).max:\n                    df[column] = df[column].astype(np.int64)  \n            else:\n                if column_min > np.finfo(np.float32).min and column_max < np.finfo(np.float32).max:\n                    df[column] = df[column].astype(np.float32)\n                else:\n                    df[column] = df[column].astype(np.float64)    \n    # Memory usage size after optimize (Mb).\n    after_size = df.memory_usage().sum() / 1024**2\n    if print_size: print('Memory usage size: before {:5.4f} Mb - after {:5.4f} Mb ({:.1f}%).'.format(before_size, after_size, 100 * (before_size - after_size) / before_size))\n    return df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# - Load data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')\ntest = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv')\norg=pd.read_csv('/kaggle/input/insurance-premium-prediction/Insurance Premium Prediction Dataset.csv')\n\ntrain.shape, test.shape, org.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"org=org[~org['Premium Amount'].isnull()]\ntrain = pd.concat([train, org], ignore_index=True)\ntrain= train.reset_index(drop=True)\ntrain.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = optimize_memory_usage(train)\ntest = optimize_memory_usage(test)\norg = optimize_memory_usage(org)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.info()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.describe().T","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Descriptive statistics of the average client: man or woman 40+ years old, with an average income of 30 thousand+ $, with two children, a car over 9 years old, with a slightly lower credit rating, insured for about 5 years**","metadata":{}},{"cell_type":"code","source":"duplicates = train.duplicated()\nprint(f\"Number of duplicates: {duplicates.sum()}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col in train.columns:\n    pct_missing = np.mean(train[col].isnull())\n    print('{} - {}%'.format(col, round(pct_missing*100)))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = test.drop(['id'], axis =1)\ntrain = train.drop(['id'], axis =1)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train.rename(columns = lambda x:re.sub('[^A-Za-z0-9]+', '_', x))\ntest = test.rename(columns = lambda x:re.sub('[^A-Za-z0-9]+', '_', x))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.Policy_Start_Date=pd.to_datetime(train.Policy_Start_Date, infer_datetime_format=True)\ntest.Policy_Start_Date=pd.to_datetime(test.Policy_Start_Date, infer_datetime_format=True)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['Policy_Start_Date:year'] = train['Policy_Start_Date'].dt.year\ntrain['Policy_Start_Date:month'] = train['Policy_Start_Date'].dt.month\ntrain['Policy_Start_Date:day'] = train['Policy_Start_Date'].dt.day\ntrain['Policy_Start_Date:day_of_week'] = train['Policy_Start_Date'].dt.day_of_week\ntrain['Policy_Start_Date:day_of_year'] = train['Policy_Start_Date'].dt.day_of_year\ntrain['Policy_Start_Date:is_year_start'] = train['Policy_Start_Date'].dt.is_year_start\ntrain['Policy_Start_Date:is_quarter_start'] = train['Policy_Start_Date'].dt.is_quarter_start\ntrain['Policy_Start_Date:is_year_end'] = train['Policy_Start_Date'].dt.is_year_end\ntrain['Policy_Start_Date:is_quarter_end'] = train['Policy_Start_Date'].dt.is_quarter_end\ntrain['Policy_Start_Date:is_month_start'] = train['Policy_Start_Date'].dt.is_month_start\ntrain['Policy_Start_Date:is_month_end'] = train['Policy_Start_Date'].dt.is_month_end\ntrain['Policy_Start_Date:is_weekend'] = np.where(train['Policy_Start_Date:day_of_week'].isin([5,6]), 1,0)\ntrain['Policy_Start_Date:hour'] = train['Policy_Start_Date'].dt.hour\ntrain['Policy_Start_Date:minute'] = train['Policy_Start_Date'].dt.minute\ntrain['Policy_Start_Date:second'] = train['Policy_Start_Date'].dt.second\ntrain['Policy_Start_Date:day_part'] = train['Policy_Start_Date:hour'].apply(day_part)\ntrain[['Policy_Start_Date:is_month_end', 'Policy_Start_Date:is_month_start',\n       'Policy_Start_Date:is_quarter_end', 'Policy_Start_Date:is_quarter_start',\n       'Policy_Start_Date:is_year_end', 'Policy_Start_Date:is_year_start']]=train[['Policy_Start_Date:is_month_end', 'Policy_Start_Date:is_month_start',\n       'Policy_Start_Date:is_quarter_end', 'Policy_Start_Date:is_quarter_start',\n       'Policy_Start_Date:is_year_end', 'Policy_Start_Date:is_year_start']].replace({True:1,False:0})\ntrain=train.drop(['Policy_Start_Date'],axis=1)\n\ntest['Policy_Start_Date:year'] = test['Policy_Start_Date'].dt.year\ntest['Policy_Start_Date:month'] = test['Policy_Start_Date'].dt.month\ntest['Policy_Start_Date:day'] = test['Policy_Start_Date'].dt.day\ntest['Policy_Start_Date:day_of_week'] = test['Policy_Start_Date'].dt.day_of_week\ntest['Policy_Start_Date:day_of_year'] = test['Policy_Start_Date'].dt.day_of_year\ntest['Policy_Start_Date:is_year_start'] = test['Policy_Start_Date'].dt.is_year_start\ntest['Policy_Start_Date:is_quarter_start'] = test['Policy_Start_Date'].dt.is_quarter_start\ntest['Policy_Start_Date:is_year_end'] = test['Policy_Start_Date'].dt.is_year_end\ntest['Policy_Start_Date:is_quarter_end'] = test['Policy_Start_Date'].dt.is_quarter_end\ntest['Policy_Start_Date:is_month_start'] = test['Policy_Start_Date'].dt.is_month_start\ntest['Policy_Start_Date:is_month_end'] = test['Policy_Start_Date'].dt.is_month_end\ntest['Policy_Start_Date:is_weekend'] = np.where(test['Policy_Start_Date:day_of_week'].isin([5,6]), 1,0)\ntest['Policy_Start_Date:hour'] = test['Policy_Start_Date'].dt.hour\ntest['Policy_Start_Date:minute'] = test['Policy_Start_Date'].dt.minute\ntest['Policy_Start_Date:second'] = test['Policy_Start_Date'].dt.second\ntest['Policy_Start_Date:day_part'] = test['Policy_Start_Date:hour'].apply(day_part)\ntest[['Policy_Start_Date:is_month_end', 'Policy_Start_Date:is_month_start',\n       'Policy_Start_Date:is_quarter_end', 'Policy_Start_Date:is_quarter_start',\n       'Policy_Start_Date:is_year_end', 'Policy_Start_Date:is_year_start']]=test[['Policy_Start_Date:is_month_end', 'Policy_Start_Date:is_month_start',\n       'Policy_Start_Date:is_quarter_end', 'Policy_Start_Date:is_quarter_start',\n       'Policy_Start_Date:is_year_end', 'Policy_Start_Date:is_year_start']].replace({True:1,False:0})\ntest=test.drop(['Policy_Start_Date'],axis=1)\n\ntrain.shape, test.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"https://www.kaggle.com/code/gauravduttakiit/org-pss4e12-dataset  - Thanks for great ideas","metadata":{}},{"cell_type":"markdown","source":"# - EDA","metadata":{}},{"cell_type":"code","source":"plot_numerical_features(train)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Data is biased. We need to tune the model so that it is possible to predict low and medium values**","metadata":{}},{"cell_type":"code","source":"plot_numerical_boxplots(train)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**The boxplot shows the distribution that there are outliers, but they are more representative of insurance premiums.**","metadata":{}},{"cell_type":"code","source":"plot_qq_plot(train)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**The quantile-quantile plot for uniform and bimodal distributions represents an S - Shaped curve. In the case of a uniform distribution, the curve spreads along the main trend line, and in the case of a bimodal distribution, it intersects it. I also draw your attention to the behavior of the letter S at zero in the case of a bimodal distribution. If the gap between the humps is large, then the quantile dependence in this place becomes almost vertical.**","metadata":{}},{"cell_type":"code","source":"# plot_pairplot(train)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_correlation_matrix(train)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_categorical_features(train)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**The data is very balanced**","metadata":{}},{"cell_type":"markdown","source":"# - Preprocess","metadata":{}},{"cell_type":"code","source":"for col in [\"Number_of_Dependents\", \"Previous_Claims\", \"Insurance_Duration\"]:\n    train[col] = train[col].fillna(0)\nfor col in train.select_dtypes(include=[np.number]).columns:\n    train[col] = train[col].fillna(train[col].mean())\nfor col in train.select_dtypes(include=['object']).columns:\n    train[col] = train[col].fillna('Unknown')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col in [\"Number_of_Dependents\", \"Previous_Claims\", \"Insurance_Duration\"]:\n    test[col] = test[col].fillna(0)\nfor col in test.select_dtypes(include=[np.number]).columns:\n    test[col] = test[col].fillna(test[col].mean())\nfor col in test.select_dtypes(include=['object']).columns:\n    test[col] = test[col].fillna('Unknown')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Q1 = train['Premium_Amount'].quantile(0.25)\nQ3 = train['Premium_Amount'].quantile(0.75)\nIQR = Q3 - Q1\ntrain = train[(train['Premium_Amount'] >= (Q1 - 1.5 * IQR)) & (train['Premium_Amount'] <= (Q3 + 1.5 * IQR))]\n\ntrain.shape, test.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"bins = [18, 35, 50, 100]\nlabels = ['18_34', '35_49', '50_100'] \n\ntrain['Age_group'] = pd.cut(train['Age'], bins=bins, labels=labels, right=False)\ntest['Age_group'] = pd.cut(test['Age'], bins=bins, labels=labels, right=False)\n\ndef categorize_score(score):\n    if score >= 849.0:\n        return 'A'\n    elif score >= 425:\n        return 'B'\n    elif score >= 212.5:\n        return 'C'\n    else:\n        return 'D'\n\ntrain['Grade_Credit_Score'] = train['Credit_Score'].apply(categorize_score)\ntest['Grade_Credit_Score'] = test['Credit_Score'].apply(categorize_score)\n\ndef categorize_income(income):\n    if income < 16948.68:\n        return 'Low'\n    elif income < 33897.36:\n        return 'Medium'\n    else:\n        return 'High'\n\ntrain['Income_category'] = train['Annual_Income'].apply(categorize_income)\ntest['Income_category'] = test['Annual_Income'].apply(categorize_income)\n\ndef categorize_health(health):\n    if health < 13.049945:\n        return 'Low'\n    elif health < 26.09989:\n        return 'Medium'\n    else:\n        return 'High'\n\ntrain['Health_category'] = train['Health_Score'].apply(categorize_health)\ntest['Health_category'] = test['Health_Score'].apply(categorize_health)\n\ntrain.shape, test.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# - VIF's","metadata":{}},{"cell_type":"code","source":"X = sm.add_constant(train.select_dtypes(include=[np.number]).iloc [:, 1:])\n\nVIFs = pd.DataFrame()\nVIFs['Variable'] = X.columns\nVIFs['VIF'] = [variance_inflation_factor(X.values, i) for i in range(X.shape[1])]\nprint(VIFs)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train.drop(columns = ['Policy_Start_Date:month', \n                              'Policy_Start_Date:day',\n                             'Policy_Start_Date:day_of_week'])\n\ntest = test.drop(columns = ['Policy_Start_Date:month',\n                            'Policy_Start_Date:day',\n                           'Policy_Start_Date:day_of_week'])\n\ntrain.shape, test.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# - Train, test preparing","metadata":{}},{"cell_type":"code","source":"X = train.drop(columns = ['Premium_Amount'])\ny = train['Premium_Amount']\n\nX_final = test\n\nX.shape, y.shape, X_final.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_cols = X.select_dtypes(include=('object')).columns\nnum_cols = X.select_dtypes(exclude=('object')).columns\n\nencoder = OneHotEncoder(sparse=False, drop='first')\nstd_scaler = StandardScaler()\n\nX_encoded = encoder.fit_transform(X[cat_cols])\nX_encoded_df = pd.DataFrame(X_encoded, columns=encoder.get_feature_names_out(cat_cols))\n\nX = pd.concat([X_encoded_df, X[num_cols].reset_index(drop=True)], axis=1)\n\nX[num_cols] = std_scaler.fit_transform(X[num_cols])\n\nX_final_encoded = encoder.transform(X_final[cat_cols])\nX_final_encoded_df = pd.DataFrame(X_final_encoded, columns=encoder.get_feature_names_out(cat_cols))\n\nX_final = pd.concat([X_final_encoded_df, X_final[num_cols].reset_index(drop=True)], axis=1)\n\nX_final[num_cols] = std_scaler.transform(X_final[num_cols])\n\nprint(f\"X shape: {X.shape}, y shape: {y.shape}, test shape: {X_final.shape}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = X.rename(columns = lambda x:re.sub('[^A-Za-z0-9]+', '_', x))\nX_final = X_final.rename(columns = lambda x:re.sub('[^A-Za-z0-9]+', '_', x))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# - Threshold","metadata":{}},{"cell_type":"code","source":"X = variance_threshold(X,0.02)\nlist_name = (X.columns)\nX_final = X_final[list_name]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X.columns","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# - Model","metadata":{}},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver() # TPU detection\nexcept ValueError:\n    tpu = None\n    gpus = tf.config.experimental.list_logical_devices(\"GPU\")\n    \nif tpu:\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu,) \n    print('Running on TPU ', tpu.cluster_spec().as_dict()['worker'])\nelif len(gpus) > 1:\n    strategy = tf.distribute.MirroredStrategy([gpu.name for gpu in gpus])\n    print('Running on multiple GPUs ', [gpu.name for gpu in gpus])\nelif len(gpus) == 1:\n    strategy = tf.distribute.get_strategy() \n    print('Running on single GPU ', gpus[0].name)\nelse:\n    strategy = tf.distribute.get_strategy() \n    print('Running on CPU')\nprint(\"Number of accelerators: \", strategy.num_replicas_in_sync)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"input_shape = [X.shape[1]]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_keras = keras.Sequential([\n    layers.BatchNormalization(input_shape=input_shape),\n    \n    layers.Flatten(),\n    layers.Dense(250, activation='relu'),\n    layers.BatchNormalization(),\n    \n    layers.Dropout(0.3),  \n    layers.Dense(250, activation='relu'),\n    layers.BatchNormalization(),\n\n    layers.Dropout(0.3),  \n    layers.Dense(250, activation='relu'),\n    layers.BatchNormalization(),\n    \n    layers.Dropout(0.3),  \n    layers.Dense(1),\n])\n\nmodel_keras.summary()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_keras_1= keras.Sequential([\n    layers.BatchNormalization(input_shape=input_shape),\n    \n    layers.Flatten(),\n    layers.Dense(12, activation='relu'),\n    layers.BatchNormalization(),\n    \n    layers.Dropout(0.3),  \n    layers.Dense(12, activation='relu'),\n    layers.BatchNormalization(),\n\n    layers.Dropout(0.3),  \n    layers.Dense(12, activation='relu'),\n    layers.BatchNormalization(),\n\n    layers.Dropout(0.3),  \n    layers.Dense(12, activation='relu'),\n    layers.BatchNormalization(),\n\n    layers.Dropout(0.3),  \n    layers.Dense(12, activation='relu'),\n    layers.BatchNormalization(),\n\n    layers.Dropout(0.3),  \n    layers.Dense(12, activation='relu'),\n    layers.BatchNormalization(),\n    \n    layers.Dropout(0.3),  \n    layers.Dense(1),\n])\n\nmodel_keras_1.summary()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_params_1 = {\n    'iterations': 1000,\n    'learning_rate': 0.1,\n    'depth': 6,\n    'loss_function': 'RMSE',\n    'random_seed': 42,\n#    'task_type': 'GPU', \n                        }\n\ncat_params_2 = {'iterations': 1108,\n                'learning_rate': 0.04971184147849109,\n                'depth': 8,\n                'l2_leaf_reg': 7.200182486855098,\n                'loss_function': 'RMSE',}\n\nxgb_params = {'n_estimators': 602, \n              'learning_rate': 0.13271234532891635,\n              'max_depth': 10, \n              'alpha': 7.038386959988065,\n              'lambda': 8.558996082105525, \n              'subsample': 0.9804490321067398, \n              'colsample_bytree': 0.958196387579288}\n\nlgbm_params = {'n_estimators': 1448, \n               'learning_rate': 0.040594995941372745, \n               'max_depth': 9, \n               'num_leaves': 61,\n               'min_child_samples': 41,\n               'subsample': 0.34277426357704066, \n               'colsample_bytree': 0.9261067677693302}","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"folds = 4\npredictions_cat, scores_cat, oof_cat = np.zeros(len(X_final)), [], np.zeros(len(X))\npredictions_xgb, scores_xgb, oof_xgb = np.zeros(len(X_final)), [], np.zeros(len(X))\npredictions_lgb, scores_lgb, oof_lgb = np.zeros(len(X_final)), [], np.zeros(len(X))\npredictions_keras, scores_keras, oof_keras = np.zeros(len(X_final)), [], np.zeros(len(X))\npredictions_keras_1, scores_keras_1, oof_keras_1 = np.zeros(len(X_final)), [], np.zeros(len(X))\npredictions_rf, scores_rf, oof_rf = np.zeros(len(X_final)), [], np.zeros(len(X))\npredictions_hgb, scores_hgb, oof_hgb = np.zeros(len(X_final)), [], np.zeros(len(X))\npredictions_cat1, scores_cat1, oof_cat1 = np.zeros(len(X_final)), [], np.zeros(len(X))\n\n\nkf = KFold(n_splits=folds, shuffle=True)\n\nfor kfold, (train_index, val_index) in enumerate(kf.split(X, y)):\n    print(f\"Training and evaluating on fold {kfold+1} out of {folds}...\")\n    tf.keras.backend.clear_session()    \n    model_keras.compile(loss='mean_squared_error',  \n                  metrics=['mean_absolute_error'],  \n                  optimizer=tf.keras.optimizers.Adam())\n    \n    X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n    y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n    \n    early_stopping = tf.keras.callbacks.EarlyStopping(patience=5,\n                                                   min_delta=0.001,\n                                                   restore_best_weights=True)\n    reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(factor=0.1, patience=2, mode='min', verbose=1)\n    \n    model_keras.fit(X_train, y_train,\n              batch_size=250,\n              epochs=30, \n              validation_data=(X_val, y_val),\n              callbacks=[reduce_lr, early_stopping])\n    \n    model_keras.save_weights(f'wg_{kfold}.weights.h5')\n    scores = model_keras.evaluate(X_val, y_val, verbose=0)\n    \n    print(scores)\n    scores_keras.append(scores)  \n    preds_val = model_keras.predict(X_val).flatten()  \n    if preds_val.shape[0] != val_index.shape[0]:\n        raise ValueError(f\"Shape mismatch for validation predictions: {preds_val.shape[0]} vs {val_index.shape[0]}\")\n    oof_keras[val_index] = preds_val \n\n    model_keras_1.compile(loss='mean_squared_error',  \n                  metrics=['mean_absolute_error'],  \n                  optimizer=tf.keras.optimizers.Adam())\n\n    early_stopping_1 = tf.keras.callbacks.EarlyStopping(patience=5,\n                                                       min_delta=0.001,\n                                                       restore_best_weights=True)\n    reduce_lr_1 = tf.keras.callbacks.ReduceLROnPlateau(factor=0.1, patience=2, mode='min', verbose=1)\n    model_keras_1.fit(X_train, y_train,\n                      batch_size=250,\n                      epochs=15, \n                      validation_data=(X_val, y_val),\n                      callbacks=[reduce_lr_1, early_stopping_1])\n    \n    model_keras_1.save_weights(f'wg_{kfold}.weights.h5')\n    scores = model_keras_1.evaluate(X_val, y_val, verbose=0)\n    \n    print(scores)\n    scores_keras_1.append(scores)  \n    preds_val_1 = model_keras_1.predict(X_val).flatten()  \n    if preds_val_1.shape[0] != val_index.shape[0]:\n        raise ValueError(f\"Shape mismatch for validation predictions: {preds_val_1.shape[0]} vs {val_index.shape[0]}\")\n    oof_keras_1[val_index] = preds_val_1 \n    \n    model_cat = CatBoostRegressor(verbose=0) \n    model_cat.fit(X_train, y_train,\n                  eval_set=(X_val, y_val), plot=False)\n    \n    oof_cat[val_index] = model_cat.predict(X_val)\n\n    \n    model_rf = RandomForestRegressor() \n    model_rf.fit(X_train, y_train)\n    \n    oof_rf[val_index] = model_rf.predict(X_val)\n\n    model_hgb = HistGradientBoostingRegressor() \n    model_hgb.fit(X_train, y_train)\n    \n    oof_hgb[val_index] = model_hgb.predict(X_val)\n    \n    model_xgb = xgb.XGBRegressor()  \n    model_xgb.fit(X_train, y_train)\n    oof_xgb[val_index] = model_xgb.predict(X_val) \n    \n    model_lgb = LGBMRegressor(**lgbm_params,verbose=-1)  \n    model_lgb.fit(X_train, y_train) \n    oof_lgb[val_index] = model_lgb.predict(X_val) \n\n    model_cat1 = CatBoostRegressor(**cat_params_2,verbose=0) \n    model_cat1.fit(X_train, y_train,\n                  eval_set=(X_val, y_val), plot=False)\n    \n    oof_cat1[val_index] = model_cat1.predict(X_val)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"results = {\n    'keras': oof_keras,\n    'keras1':oof_keras_1,\n    'cat': oof_cat,\n    'xgb': oof_xgb,\n    'lgb': oof_lgb,\n    'rf': oof_rf,\n    'hgb': oof_hgb,\n    'cat': oof_cat\n}\n\ndef objective(trial):\n    weights = []\n    for _ in range(len(results)):  \n        weights.append(trial.suggest_uniform('weight_{}'.format(_), 0, 1))\n    weights = np.array(weights)\n    \n    weights /= np.sum(weights) if np.sum(weights) != 0 else 1  \n\n    y_pred_weighted = np.zeros(len(y), dtype=float)  \n    for i, name in enumerate(results.keys()):\n        oof_predictions = results[name]\n        \n        if oof_predictions.shape[0] != y_pred_weighted.shape[0]:\n            raise ValueError(f\"Shape mismatch for {name}: {oof_predictions.shape[0]} vs {y_pred_weighted.shape[0]}\")\n\n        y_pred_weighted += weights[i] * oof_predictions  \n\n    metric = mean_squared_error(y, y_pred_weighted) \n    return metric\n\nstudy = optuna.create_study(direction='minimize')  \nstudy.optimize(objective, n_trials=2000, timeout=600)\n\nbest_trial = study.best_trial\noptimal_weights = best_trial.params\n\nprint(\"Optimal weights:\", optimal_weights)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred_test_final = np.zeros(len(X_final))\n\nmodels = [model_keras, model_keras_1, model_cat, model_rf, model_hgb, model_xgb, model_lgb, model_cat1]\n\nfor i, model in enumerate(models):\n    y_pred_test = model.predict(X_final)\n    \n    if model is model_keras:\n        y_pred_test = y_pred_test.flatten()\n    elif model is model_keras_1:\n        y_pred_test = y_pred_test.flatten()\n\n    y_pred_test_final += optimal_weights['weight_{}'.format(i)] * y_pred_test\n\nprint(\"Final predictions:\", y_pred_test_final)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# - Submit","metadata":{}},{"cell_type":"code","source":"sample = pd.read_csv('/kaggle/input/playground-series-s4e12/sample_submission.csv')\nsample['Premium Amount'] = y_pred_test_final\nsample.to_csv('submission.csv', index=False)\nsample.head(10)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}