{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# HyperParameterTraining for ANN model - CAFA 5","metadata":{"execution":{"iopub.status.busy":"2023-06-27T15:01:36.033919Z","iopub.execute_input":"2023-06-27T15:01:36.034378Z","iopub.status.idle":"2023-06-27T15:01:36.038621Z","shell.execute_reply.started":"2023-06-27T15:01:36.034343Z","shell.execute_reply":"2023-06-27T15:01:36.037774Z"}}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\n\nimport numpy as np\nimport pandas as pd\n\nimport tensorflow as tf\n\nimport h5py\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import hamming_loss\n\n!pip install -U -q keras-tuner\nimport keras_tuner as kt","metadata":{"execution":{"iopub.status.busy":"2023-06-28T18:37:00.04361Z","iopub.execute_input":"2023-06-28T18:37:00.044502Z","iopub.status.idle":"2023-06-28T18:37:23.565052Z","shell.execute_reply.started":"2023-06-28T18:37:00.044454Z","shell.execute_reply":"2023-06-28T18:37:23.563979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if tf.test.gpu_device_name():\n    print(\"GPU is available : {}\".format(tf.test.gpu_device_name()))\nelse:\n    print(\"GPU is not available, Training CPU instead.\")","metadata":{"execution":{"iopub.status.busy":"2023-06-28T18:37:55.103223Z","iopub.execute_input":"2023-06-28T18:37:55.104085Z","iopub.status.idle":"2023-06-28T18:37:59.409537Z","shell.execute_reply.started":"2023-06-28T18:37:55.104031Z","shell.execute_reply":"2023-06-28T18:37:59.408558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-28T18:38:02.058054Z","iopub.execute_input":"2023-06-28T18:38:02.058744Z","iopub.status.idle":"2023-06-28T18:38:02.089249Z","shell.execute_reply.started":"2023-06-28T18:38:02.058709Z","shell.execute_reply":"2023-06-28T18:38:02.088299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Just to involve the CAFA 5 dataset\n# train_tax = pd.read_csv('/kaggle/input/cafa-5-protein-function-prediction/Train/train_taxonomy.tsv',sep='\\t')\n# train_tax.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-27T15:22:37.168334Z","iopub.execute_input":"2023-06-27T15:22:37.168791Z","iopub.status.idle":"2023-06-27T15:22:37.302608Z","shell.execute_reply.started":"2023-06-27T15:22:37.168759Z","shell.execute_reply":"2023-06-27T15:22:37.301425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = np.load('/kaggle/input/t5embeds/train_embeds.npy')\n\nwith h5py.File('/kaggle/input/train-labels-cafa5/train_labels.h5', 'r') as hf:\n    Y = hf['labels'][:]","metadata":{"execution":{"iopub.status.busy":"2023-06-28T18:38:09.100947Z","iopub.execute_input":"2023-06-28T18:38:09.10178Z","iopub.status.idle":"2023-06-28T18:38:21.264893Z","shell.execute_reply.started":"2023-06-28T18:38:09.101742Z","shell.execute_reply":"2023-06-28T18:38:21.263849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size= 0.01, random_state=123)","metadata":{"execution":{"iopub.status.busy":"2023-06-28T18:38:29.379749Z","iopub.execute_input":"2023-06-28T18:38:29.380102Z","iopub.status.idle":"2023-06-28T18:38:29.827051Z","shell.execute_reply.started":"2023-06-28T18:38:29.380072Z","shell.execute_reply":"2023-06-28T18:38:29.82606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check the hamming loss on the [Previous ANN Model](https://www.kaggle.com/code/stomar21/kerastuner?scriptVersionId=135047119)","metadata":{}},{"cell_type":"code","source":"old_model = tf.keras.models.load_model('/kaggle/input/train-labels-cafa5/KerasTunerMLP.h5')\npred_old = np.round(old_model.predict(X_test))\npred_old_train = np.round(old_model.predict(X_train))\n\nprint('Hamming Loss for Old Model on Test Set= {}'.format(hamming_loss(Y_test,pred_old)))\nprint('Hamming Loss for Old Model on Train Set= {}'.format(hamming_loss(Y_train,pred_old_train)))","metadata":{"execution":{"iopub.status.busy":"2023-06-28T18:39:18.920514Z","iopub.execute_input":"2023-06-28T18:39:18.920888Z","iopub.status.idle":"2023-06-28T18:40:04.996152Z","shell.execute_reply.started":"2023-06-28T18:39:18.920857Z","shell.execute_reply":"2023-06-28T18:40:04.993995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"```\nOutput - \n------------------------------------------------------------------------\n45/45 [==============================] - 3s 2ms/step\n4401/4401 [==============================] - 8s 2ms/step\nHamming Loss for Old Model on Test Set= 0.017841649098149448\nHamming Loss for Old Model on Train Set= 0.016263271387959827\n------------------------------------------------------------------------\n```","metadata":{}},{"cell_type":"markdown","source":"# Creating A Keras Tuner HyperBand to look for best hyperparameters\n(Code is now Commented Out as the notebook was then used for kaggle submission)","metadata":{}},{"cell_type":"code","source":"# Creating a hamming_loss function for passing as a metric\ndef hamming_loss_tf(y_true, y_pred):\n    y_pred_rounded = tf.round(y_pred)\n    return tf.py_function(hamming_loss, (y_true, y_pred_rounded), tf.float32)","metadata":{"execution":{"iopub.status.busy":"2023-06-28T18:48:45.625185Z","iopub.execute_input":"2023-06-28T18:48:45.625901Z","iopub.status.idle":"2023-06-28T18:48:45.630731Z","shell.execute_reply.started":"2023-06-28T18:48:45.625863Z","shell.execute_reply":"2023-06-28T18:48:45.629762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf keras_tuner_directory","metadata":{"execution":{"iopub.status.busy":"2023-06-28T18:51:47.06864Z","iopub.execute_input":"2023-06-28T18:51:47.069002Z","iopub.status.idle":"2023-06-28T18:51:48.106124Z","shell.execute_reply.started":"2023-06-28T18:51:47.068972Z","shell.execute_reply":"2023-06-28T18:51:48.104793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tuner = kt.Hyperband(model_builder,\n                     objective=kt.Objective(\"val_hamming_loss_tf\", direction=\"min\"),\n                     max_epochs=10,\n                     factor=3,\n                     directory='keras_tuner_directory',\n                     project_name='keras_optimisation_MLP_hamming_loss')","metadata":{"execution":{"iopub.status.busy":"2023-06-28T18:51:48.108947Z","iopub.execute_input":"2023-06-28T18:51:48.109473Z","iopub.status.idle":"2023-06-28T18:51:48.188322Z","shell.execute_reply.started":"2023-06-28T18:51:48.109412Z","shell.execute_reply":"2023-06-28T18:51:48.187497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stop_early = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3)","metadata":{"execution":{"iopub.status.busy":"2023-06-28T18:51:49.001597Z","iopub.execute_input":"2023-06-28T18:51:49.002108Z","iopub.status.idle":"2023-06-28T18:51:49.019369Z","shell.execute_reply.started":"2023-06-28T18:51:49.002061Z","shell.execute_reply":"2023-06-28T18:51:49.014654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tuner.search(X_train, Y_train, epochs=25, validation_split=0.2, callbacks=[stop_early])","metadata":{"execution":{"iopub.status.busy":"2023-06-28T18:51:49.554549Z","iopub.execute_input":"2023-06-28T18:51:49.554899Z","iopub.status.idle":"2023-06-28T20:11:08.024355Z","shell.execute_reply.started":"2023-06-28T18:51:49.554869Z","shell.execute_reply":"2023-06-28T20:11:08.023327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_hps = tuner.get_best_hyperparameters(num_trials=1)[0]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Best Parameters After Search')\nfor hp_name, hp_value in best_hps.values.items():\n    print(f'{hp_name}: {hp_value}')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"```\nOuput - \n------------------------------------\nBest Parameters After Search\nactivation: relu\nlayer_1: 1024\nlayer_2: 256\nlayer_3: 512\nlearning_rate: 0.001\ntuner/epochs: 10\ntuner/initial_epoch: 4\ntuner/bracket: 2\ntuner/round: 2\ntuner/trial_id: 0014\n------------------------------------\n```","metadata":{}},{"cell_type":"code","source":"model = tuner.hypermodel.build(best_hps)\nhistory = model.fit(X_train, y_train, epochs=25, validation_data=(X_test,Y_test),\n                    callbacks=[stop_early])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('KerasTunerMLP_hamming_loss.h5')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Checking the new model","metadata":{}},{"cell_type":"code","source":"new_model = tf.keras.models.load_model('/kaggle/input/train-labels-cafa5/KerasTunerMLP_hamming_loss.h5')\npred_new = new_model.predict(X_test)\npred_new[pred_new<=0.5] = 0\npred_new[pred_new>0.5] = 1\npred_new_train = new_model.predict(X_train)\npred_new_train[pred_new_train<=0.5] = 0\npred_new_train[pred_new_train>0.5] = 1\n\nprint('Hamming Loss for New Model on Test Set= {}'.format(hamming_loss(Y_test,pred_new)))\nprint('Hamming Loss for New Model on Train Set= {}'.format(hamming_loss(Y_train,pred_new_train)))","metadata":{"execution":{"iopub.status.busy":"2023-06-27T15:02:09.14791Z","iopub.execute_input":"2023-06-27T15:02:09.148422Z","iopub.status.idle":"2023-06-27T15:02:51.962225Z","shell.execute_reply.started":"2023-06-27T15:02:09.148375Z","shell.execute_reply":"2023-06-27T15:02:51.961074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"```\nOutput - \n------------------------------------------------------------------------\n45/45 [==============================] - 0s 5ms/step\n4401/4401 [==============================] - 21s 5ms/step\nHamming Loss for New Model on Test Set= 0.017841649098149448\nHamming Loss for New Model on Train Set= 0.016263271387959827\n------------------------------------------------------------------------\n```","metadata":{}},{"cell_type":"markdown","source":"# Creating A New Submission Using the New Model","metadata":{}},{"cell_type":"code","source":"# test_embeddings = np.load('/kaggle/input/t5embeds/test_embeds.npy')\n# labels = np.load('/kaggle/input/train-labels-cafa5/top_1500_labels.npy',allow_pickle=True)\n# new_model = tf.keras.models.load_model('/kaggle/input/train-labels-cafa5/KerasTunerMLP.h5')\n# print('Data Loaded Successfully!')","metadata":{"execution":{"iopub.status.busy":"2023-06-27T15:16:26.07292Z","iopub.execute_input":"2023-06-27T15:16:26.074702Z","iopub.status.idle":"2023-06-27T15:16:37.895504Z","shell.execute_reply.started":"2023-06-27T15:16:26.074628Z","shell.execute_reply":"2023-06-27T15:16:37.894209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# predictions = new_model.predict(test_embeddings)","metadata":{"execution":{"iopub.status.busy":"2023-06-27T15:16:58.32216Z","iopub.execute_input":"2023-06-27T15:16:58.322587Z","iopub.status.idle":"2023-06-27T15:17:40.585811Z","shell.execute_reply.started":"2023-06-27T15:16:58.322553Z","shell.execute_reply":"2023-06-27T15:17:40.584131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %time\n# test_protein_ids = np.load('/kaggle/input/t5embeds/test_ids.npy')\n# l = []\n# for k in list(test_protein_ids):\n#     l += [ k] * predictions.shape[1]","metadata":{"execution":{"iopub.status.busy":"2023-06-27T15:18:02.105563Z","iopub.execute_input":"2023-06-27T15:18:02.106043Z","iopub.status.idle":"2023-06-27T15:18:05.025222Z","shell.execute_reply.started":"2023-06-27T15:18:02.10601Z","shell.execute_reply":"2023-06-27T15:18:05.023936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# df_submission = pd.DataFrame(\n#     {\n#         'Protein ID': l,\n#         'GO Term ID': np.tile(labels, predictions.shape[0]),\n#         'Prediction': np.round(predictions.ravel(),3)\n#     }\n# )\n\n# print('df_submission Created Successfully!')","metadata":{"execution":{"iopub.status.busy":"2023-06-27T15:18:55.089487Z","iopub.execute_input":"2023-06-27T15:18:55.090424Z","iopub.status.idle":"2023-06-27T15:19:23.338898Z","shell.execute_reply.started":"2023-06-27T15:18:55.090383Z","shell.execute_reply":"2023-06-27T15:19:23.33799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_submission.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-27T15:19:30.882644Z","iopub.execute_input":"2023-06-27T15:19:30.883073Z","iopub.status.idle":"2023-06-27T15:19:30.917814Z","shell.execute_reply.started":"2023-06-27T15:19:30.883039Z","shell.execute_reply":"2023-06-27T15:19:30.916869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# print('Starting to save submission to .csv . . . . .')\n# df_submission.to_csv('submission.tsv', sep='\\t', header=None, index= None)\n# print('Done!\\nCreated a submission.tsv file!')","metadata":{},"execution_count":null,"outputs":[]}]}