{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.15","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# S4E12 keras w/TPU","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install --upgrade pip\n!pip install --upgrade tensorflow  # Upgrade to the latest version\n!pip install --upgrade keras\n!pip install scikit-learn scikeras","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T03:49:32.851815Z","iopub.execute_input":"2024-12-08T03:49:32.852289Z","iopub.status.idle":"2024-12-08T03:49:40.136782Z","shell.execute_reply.started":"2024-12-08T03:49:32.852253Z","shell.execute_reply":"2024-12-08T03:49:40.135755Z"},"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip list | grep keras\n!pip list | grep tensorflow","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# set up TPU","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras import layers, models\nimport numpy as np\n\n# Reset TensorFlow session\ntf.keras.backend.clear_session()\ntf.config.optimizer.set_experimental_options({\"layout_optimizer\": False, \"constant_folding\": False})\n\n# Set up TPU\ntry:\n    resolver = tf.distribute.cluster_resolver.TPUClusterResolver()  # Resolving TPU\n    tf.config.experimental_connect_to_cluster(resolver)  # Connect to TPU\n    strategy = tf.distribute.TPUStrategy(resolver)  # TPU strategy\n    print(\"TPU devices:\", resolver.cluster_spec().as_dict())\n    \nexcept ValueError:\n    strategy = tf.distribute.get_strategy()  # Use default strategy (CPU or GPU) if TPU is unavailable\n    print(\"Using default strategy (CPU or GPU)\")\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T03:49:40.138957Z","iopub.execute_input":"2024-12-08T03:49:40.139265Z","iopub.status.idle":"2024-12-08T03:49:40.535312Z","shell.execute_reply.started":"2024-12-08T03:49:40.139236Z","shell.execute_reply":"2024-12-08T03:49:40.534467Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.model_selection import KFold, train_test_split\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import mean_squared_log_error","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T03:49:40.536316Z","iopub.execute_input":"2024-12-08T03:49:40.536586Z","iopub.status.idle":"2024-12-08T03:49:40.547004Z","shell.execute_reply.started":"2024-12-08T03:49:40.536559Z","shell.execute_reply":"2024-12-08T03:49:40.546266Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Processing","metadata":{}},{"cell_type":"code","source":"train0 = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\")\ntest0 = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T03:49:40.548044Z","iopub.execute_input":"2024-12-08T03:49:40.548294Z","iopub.status.idle":"2024-12-08T03:49:45.940871Z","shell.execute_reply.started":"2024-12-08T03:49:40.54827Z","shell.execute_reply":"2024-12-08T03:49:45.940003Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\ndef labelencoder(df_train, df_test):\n    combined_df = pd.concat([df_train, df_test])\n    for c in combined_df.columns:\n        if combined_df[c].dtype == 'object':\n            combined_df[c] = combined_df[c].fillna('N')\n            lbl = LabelEncoder()\n            lbl.fit(list(combined_df[c].values))\n            combined_df[c] = lbl.transform(combined_df[c].values)\n    return combined_df.iloc[:len(df_train)], combined_df.iloc[len(df_train):]\n\ntrain0,test0=labelencoder(train0,test0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T03:49:45.943158Z","iopub.execute_input":"2024-12-08T03:49:45.94347Z","iopub.status.idle":"2024-12-08T03:50:08.655486Z","shell.execute_reply.started":"2024-12-08T03:49:45.943442Z","shell.execute_reply":"2024-12-08T03:50:08.65447Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train=train0.fillna(train0.mean())\ntest=test0.fillna(test0.mean())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T03:50:08.656647Z","iopub.execute_input":"2024-12-08T03:50:08.657151Z","iopub.status.idle":"2024-12-08T03:50:09.281952Z","shell.execute_reply.started":"2024-12-08T03:50:08.657119Z","shell.execute_reply":"2024-12-08T03:50:09.281108Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target = train['Premium Amount']\ndata = train.drop(['Premium Amount'],axis=1)\ntest = test.drop(['Premium Amount'],axis=1)\n\ncolumns=data.columns.to_list()\nprint(columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T03:50:09.283165Z","iopub.execute_input":"2024-12-08T03:50:09.283476Z","iopub.status.idle":"2024-12-08T03:50:09.513515Z","shell.execute_reply.started":"2024-12-08T03:50:09.283446Z","shell.execute_reply":"2024-12-08T03:50:09.512868Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"x_train, x_test, y_train, y_test = train_test_split(data, target, test_size=0.2,random_state=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T03:50:09.514729Z","iopub.execute_input":"2024-12-08T03:50:09.515057Z","iopub.status.idle":"2024-12-08T03:50:09.886671Z","shell.execute_reply.started":"2024-12-08T03:50:09.515022Z","shell.execute_reply":"2024-12-08T03:50:09.885766Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(x_train.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T03:50:09.887667Z","iopub.execute_input":"2024-12-08T03:50:09.887948Z","iopub.status.idle":"2024-12-08T03:50:09.891853Z","shell.execute_reply.started":"2024-12-08T03:50:09.887923Z","shell.execute_reply":"2024-12-08T03:50:09.891199Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# set model in strategy.scope","metadata":{}},{"cell_type":"code","source":"def rmsle_metric(y_true, y_pred):\n    y_true = tf.keras.backend.clip(y_true, tf.keras.backend.epsilon(), None)\n    y_pred = tf.keras.backend.clip(y_pred, tf.keras.backend.epsilon(), None)\n    log_true = tf.math.log(y_true + 1.0)\n    log_pred = tf.math.log(y_pred + 1.0)\n    return tf.sqrt(tf.reduce_mean(tf.square(log_true - log_pred)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T03:50:09.892698Z","iopub.execute_input":"2024-12-08T03:50:09.892949Z","iopub.status.idle":"2024-12-08T03:50:09.901421Z","shell.execute_reply.started":"2024-12-08T03:50:09.892925Z","shell.execute_reply":"2024-12-08T03:50:09.900845Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#from keras.wrappers.scikit_learn import KerasRegressor\n#from tensorflow.keras.wrappers.scikit_learn import KerasRegressor\nfrom scikeras.wrappers import KerasRegressor\n\nwith strategy.scope():\n    build_model = models.Sequential([\n        layers.Input(shape=(20,)),\n        layers.Dense(128, activation='relu'),\n        layers.BatchNormalization(),\n        layers.Dense(64, activation='relu'),\n        layers.BatchNormalization(),\n        layers.Dense(1)\n    ])\n        \n    build_model.compile(optimizer='adam',\n                  loss='mean_squared_error',  # Use appropriate loss for regression\n                          metrics=[rmsle_metric]  # RMSLE \n                 )\n\nmodel = KerasRegressor(build_fn=build_model, verbose=0)\n\nmodel.fit(x_train, y_train, epochs=30, batch_size=64, validation_data=(x_test, y_test))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T03:50:09.902355Z","iopub.execute_input":"2024-12-08T03:50:09.9026Z","iopub.status.idle":"2024-12-08T03:51:40.748016Z","shell.execute_reply.started":"2024-12-08T03:50:09.902575Z","shell.execute_reply":"2024-12-08T03:51:40.747083Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"    model = models.Sequential([\n        layers.Input(shape=(20,)),  # Define input shape explicitly\n        layers.Dense(128, activation='relu'),\n        layers.Dense(1)  # No activation for regression\n    ])","metadata":{}},{"cell_type":"markdown","source":"    model = models.Sequential([\n        layers.Input(shape=(20,)),\n        layers.Dense(128, activation='relu'),\n        layers.Dense(64, activation='relu'),\n        layers.Dense(32, activation='relu'),\n        layers.Dense(1)  # No activation for regression\n    ])","metadata":{"execution":{"iopub.status.busy":"2024-12-08T03:50:09.902355Z","iopub.execute_input":"2024-12-08T03:50:09.9026Z","iopub.status.idle":"2024-12-08T03:51:40.748016Z","shell.execute_reply.started":"2024-12-08T03:50:09.902575Z","shell.execute_reply":"2024-12-08T03:51:40.747083Z"}}},{"cell_type":"markdown","source":"    model = models.Sequential([\n        layers.Input(shape=(20,)),\n        layers.Dense(128, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(0.01)),\n        layers.Dropout(0.2),  # 20% dropout\n        layers.Dense(64, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(0.01)),\n        layers.Dropout(0.2),\n        layers.Dense(1)\n    ])","metadata":{}},{"cell_type":"markdown","source":"    model = models.Sequential([\n        layers.Input(shape=(20,)),\n        layers.Dense(128, activation='relu'),\n        layers.BatchNormalization(),\n        layers.Dense(64, activation='relu'),\n        layers.BatchNormalization(),\n        layers.Dense(1)\n    ])\n    ","metadata":{}},{"cell_type":"markdown","source":"\n    model = models.Sequential([\n        layers.Input(shape=(20,)),\n        layers.Dense(128),\n        layers.LeakyReLU(alpha=0.1),\n        layers.Dense(64),\n        layers.LeakyReLU(alpha=0.1),\n        layers.Dense(1)\n    ])","metadata":{}},{"cell_type":"code","source":"preds = np.zeros(test[columns].shape[0])\nkf = KFold(n_splits=5,random_state=48,shuffle=True)\nfor trn_idx, test_idx in kf.split(train[columns],target):\n    X_tr,X_val=train[columns].iloc[trn_idx],train[columns].iloc[test_idx]\n    y_tr,y_val=target.iloc[trn_idx],target.iloc[test_idx]\n\n    model.fit(X_tr,y_tr)\n    preds+=model.predict(test[columns])/kf.n_splits   \n\n    y_val_pred = model.predict(X_val)\n\n    rmsle = np.sqrt(mean_squared_log_error(abs(y_val), abs(y_val_pred))) \n    print(f'RMSLE for fold: {rmsle}')\n    \n    #rmse = mean_squared_error(y_val, y_val_pred,squared=False)\n    #print(rmse)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-08T03:51:40.749391Z","iopub.execute_input":"2024-12-08T03:51:40.749688Z","iopub.status.idle":"2024-12-08T03:51:40.755268Z","shell.execute_reply.started":"2024-12-08T03:51:40.749658Z","shell.execute_reply":"2024-12-08T03:51:40.754538Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"    preds = np.zeros(test[columns].shape[0])  # Initialize with the same length as the number of test samples\n    kf = KFold(n_splits=5, random_state=48, shuffle=True)\n    \n    for trn_idx, test_idx in kf.split(train[columns], target):\n        X_tr, X_val = train[columns].iloc[trn_idx], train[columns].iloc[test_idx]\n        y_tr, y_val = target.iloc[trn_idx], target.iloc[test_idx]\n    \n        # Train the model\n        model.fit(X_tr, np.log1p(y_tr))  \n    \n        # Predict on test data\n        preds_fold = np.expm1(model.predict(test[columns]))  \n        preds_fold = np.nan_to_num(preds_fold, nan=0, posinf=0, neginf=0)  \n        preds_fold = np.clip(preds_fold, 0, 1e10)\n    \n        # Accumulate predictions\n        preds += preds_fold.flatten() / kf.n_splits  # Use .flatten() to ensure 1D shape\n    \n        # Predict on validation data\n        y_val_pred = np.expm1(model.predict(X_val))  \n        y_val_pred = np.nan_to_num(y_val_pred, nan=0, posinf=0, neginf=0)  \n        y_val_pred = np.clip(y_val_pred, 0, 1e10)\n    \n        # Actual validation values\n        y_val_actual = np.expm1(y_val)  \n        y_val_actual = np.nan_to_num(y_val_actual, nan=0, posinf=0, neginf=0) \n    \n        # Calculate RMSLE\n        try:\n            rmsle = np.sqrt(mean_squared_log_error(y_val_actual, y_val_pred)) \n            print(f'RMSLE for fold: {rmsle}')\n        except ValueError as e:\n            print(f\"Error in fold: {e}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-12-08T03:51:41.041855Z","iopub.execute_input":"2024-12-08T03:51:41.042118Z"}}},{"cell_type":"code","source":"pd.Series(preds).value_counts()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample = pd.read_csv(\"/kaggle/input/playground-series-s4e12/sample_submission.csv\")\nprint(sample.shape)\nsubm = sample\nsubm['Premium Amount'] = preds.astype(int)\nsubm.to_csv('submission.csv',index=False)\nsubm","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"    The strategy.scope() approach is required for using TPU, but it works with CPU and GPU as well without any changes to the code. The strategy.scope() is used to define the scope in which the model is created and managed for distributed training, but TensorFlow will handle it differently based on the available hardware.\n    \n    Here’s how it works:\n    \n    TPU: TPUStrategy distributes the model training across TPU cores.\n    \n    GPU: If you use GPU instead of TPU, TensorFlow will use a MirroredStrategy (or another strategy suitable for multi-GPU setup) to distribute the training across available GPU devices.\n    \n    CPU: If you use CPU, TensorFlow defaults to using a single device, but it still works within the strategy.scope() to maintain consistency and ensure the code works in distributed environments if needed in the future.","metadata":{}}]}