{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import tensorflow as tf\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n# for progressbar widget\nimport progressbar","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-25T08:49:16.269075Z","iopub.execute_input":"2023-07-25T08:49:16.269439Z","iopub.status.idle":"2023-07-25T08:49:25.269943Z","shell.execute_reply.started":"2023-07-25T08:49:16.269411Z","shell.execute_reply":"2023-07-25T08:49:25.268865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check versions\nprint(\"TensorFlow v\" + tf.__version__)\nprint(\"Numpy v\" + np.__version__)","metadata":{"execution":{"iopub.status.busy":"2023-07-25T08:49:25.272068Z","iopub.execute_input":"2023-07-25T08:49:25.273397Z","iopub.status.idle":"2023-07-25T08:49:25.279849Z","shell.execute_reply.started":"2023-07-25T08:49:25.273355Z","shell.execute_reply":"2023-07-25T08:49:25.278713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load Dataset\ntrain_terms = pd.read_csv(\"/kaggle/input/cafa-5-protein-function-prediction/Train/train_terms.tsv\",sep=\"\\t\")\nprint(train_terms.shape)","metadata":{"execution":{"iopub.status.busy":"2023-07-25T08:49:25.281465Z","iopub.execute_input":"2023-07-25T08:49:25.282218Z","iopub.status.idle":"2023-07-25T08:49:28.461436Z","shell.execute_reply.started":"2023-07-25T08:49:25.282179Z","shell.execute_reply":"2023-07-25T08:49:28.460326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_terms.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-25T08:49:28.463754Z","iopub.execute_input":"2023-07-25T08:49:28.464098Z","iopub.status.idle":"2023-07-25T08:49:28.484928Z","shell.execute_reply.started":"2023-07-25T08:49:28.464067Z","shell.execute_reply":"2023-07-25T08:49:28.48379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load the pre-calculated protein embeddings as a np array\ntrain_protein_ids = np.load('/kaggle/input/t5embeds/train_ids.npy')\nprint(train_protein_ids.shape)","metadata":{"execution":{"iopub.status.busy":"2023-07-25T08:49:28.488177Z","iopub.execute_input":"2023-07-25T08:49:28.488523Z","iopub.status.idle":"2023-07-25T08:49:28.541021Z","shell.execute_reply.started":"2023-07-25T08:49:28.488493Z","shell.execute_reply":"2023-07-25T08:49:28.540061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_protein_ids[:5]","metadata":{"execution":{"iopub.status.busy":"2023-07-25T08:49:28.542267Z","iopub.execute_input":"2023-07-25T08:49:28.542698Z","iopub.status.idle":"2023-07-25T08:49:28.549532Z","shell.execute_reply.started":"2023-07-25T08:49:28.542668Z","shell.execute_reply":"2023-07-25T08:49:28.548429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_embeddings = np.load('/kaggle/input/t5embeds/train_embeds.npy')\ntrain_embeddings.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-25T08:49:28.550896Z","iopub.execute_input":"2023-07-25T08:49:28.55124Z","iopub.status.idle":"2023-07-25T08:49:39.020781Z","shell.execute_reply.started":"2023-07-25T08:49:28.551205Z","shell.execute_reply":"2023-07-25T08:49:39.019973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# convert embeddings np array (train_embeddings) into a pd dataframe\ncolumn_num = train_embeddings.shape[1]\ntrain_df = pd.DataFrame(train_embeddings, columns = [\"Column_\" + str(i) for i in range(1, column_num + 1)])\nprint(train_df.shape)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-25T08:49:39.022014Z","iopub.execute_input":"2023-07-25T08:49:39.022706Z","iopub.status.idle":"2023-07-25T08:49:39.031803Z","shell.execute_reply.started":"2023-07-25T08:49:39.022666Z","shell.execute_reply":"2023-07-25T08:49:39.029982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df contains the embeddings\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-25T08:49:39.032894Z","iopub.execute_input":"2023-07-25T08:49:39.033194Z","iopub.status.idle":"2023-07-25T08:49:39.073595Z","shell.execute_reply.started":"2023-07-25T08:49:39.033168Z","shell.execute_reply":"2023-07-25T08:49:39.072698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# prepare the dataset\n\n# Select first 1500 values for plotting (for simplication since there are more than 4,000 lables)\nplot_df = train_terms['term'].value_counts().iloc[:100]\n\nfigure, axis = plt.subplots(1, 1, figsize = (12, 6))\n\nbp = sns.barplot(ax=axis, x=np.array(plot_df.index), y=plot_df.values)\nbp.set_xticklabels(bp.get_xticklabels(), rotation=90, size = 6)\naxis.set_title('Top 100 frequent GO term IDs')\nbp.set_xlabel(\"GO term IDs\", fontsize = 12)\nbp.set_ylabel(\"Count\", fontsize = 12)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-25T08:49:39.077377Z","iopub.execute_input":"2023-07-25T08:49:39.078044Z","iopub.status.idle":"2023-07-25T08:49:40.710908Z","shell.execute_reply.started":"2023-07-25T08:49:39.078008Z","shell.execute_reply":"2023-07-25T08:49:40.709811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#help(sns.barplot)\n#help(plt.subplots)","metadata":{"execution":{"iopub.status.busy":"2023-07-25T08:49:40.712159Z","iopub.execute_input":"2023-07-25T08:49:40.712556Z","iopub.status.idle":"2023-07-25T08:49:40.717262Z","shell.execute_reply.started":"2023-07-25T08:49:40.712452Z","shell.execute_reply":"2023-07-25T08:49:40.716025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set the limit for label\nnum_of_labels = 1500\n\n# Take value counts in descending order and fetch first 1500 'GO term ID' as labels\nlabels = train_terms['term'].value_counts().index[:num_of_labels].tolist()\n#labels","metadata":{"execution":{"iopub.status.busy":"2023-07-25T08:49:40.718572Z","iopub.execute_input":"2023-07-25T08:49:40.718953Z","iopub.status.idle":"2023-07-25T08:49:41.248262Z","shell.execute_reply.started":"2023-07-25T08:49:40.718917Z","shell.execute_reply":"2023-07-25T08:49:41.246887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make a new datafram by filtering the train terms with selected 'GO Term ID'\n\n# Fetch the train_terms data for the relevant labels only\ntrain_terms_updated = train_terms.loc[train_terms['term'].isin(labels)]\n","metadata":{"execution":{"iopub.status.busy":"2023-07-25T08:49:41.249892Z","iopub.execute_input":"2023-07-25T08:49:41.25021Z","iopub.status.idle":"2023-07-25T08:49:41.904601Z","shell.execute_reply.started":"2023-07-25T08:49:41.250182Z","shell.execute_reply":"2023-07-25T08:49:41.903795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_terms_updated.head","metadata":{"execution":{"iopub.status.busy":"2023-07-25T08:49:41.905634Z","iopub.execute_input":"2023-07-25T08:49:41.906155Z","iopub.status.idle":"2023-07-25T08:49:41.916807Z","shell.execute_reply.started":"2023-07-25T08:49:41.906122Z","shell.execute_reply":"2023-07-25T08:49:41.915934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the aspect values in the new train_terms_updated datafram using a pie chart\n\npie_df = train_terms_updated['aspect'].value_counts()\npalette_color = sns.color_palette('bright')\nplt.pie(pie_df.values, labels=np.array(pie_df.index), colors=palette_color, autopct='%.0f%%')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-25T08:49:41.917868Z","iopub.execute_input":"2023-07-25T08:49:41.918149Z","iopub.status.idle":"2023-07-25T08:49:42.386801Z","shell.execute_reply.started":"2023-07-25T08:49:41.918125Z","shell.execute_reply":"2023-07-25T08:49:42.385268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_terms_updated.loc[:, 'has_label'] = 1\ndf_pivot = train_terms_updated[train_terms_updated.term.isin(labels)].pivot('EntryID', 'term', 'has_label').fillna(0).astype(int)\nlabels_df = df_pivot.loc[train_protein_ids, labels]","metadata":{"execution":{"iopub.status.busy":"2023-07-25T08:49:42.388194Z","iopub.execute_input":"2023-07-25T08:49:42.390657Z","iopub.status.idle":"2023-07-25T08:49:50.512466Z","shell.execute_reply.started":"2023-07-25T08:49:42.390612Z","shell.execute_reply":"2023-07-25T08:49:50.511421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Setup progressbar settings.\n# This is strictly for aesthetic.\nbar = progressbar.ProgressBar(maxval=num_of_labels, \\\n    widgets=[progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage()])\n\n# Create an empty dataframe of required size for storing the labels,\n# i.e, train_size x num_of_labels (142246 x 1500)\ntrain_size = train_protein_ids.shape[0] # len(X)\ntrain_labels = np.zeros((train_size ,num_of_labels))\n\n# Convert from numpy to pandas series for better handling\nseries_train_protein_ids = pd.Series(train_protein_ids)\n\n# Loop through each label\nfor i in range(num_of_labels):\n    # For each label, fetch the corresponding train_terms data\n    n_train_terms = train_terms_updated[train_terms_updated['term'] ==  labels[i]]\n    \n    # Fetch all the unique EntryId aka proteins related to the current label(GO term ID)\n    label_related_proteins = n_train_terms['EntryID'].unique()\n    \n    # In the series_train_protein_ids pandas series, if a protein is related\n    # to the current label, then mark it as 1, else 0.\n    # Replace the ith column of train_Y with with that pandas series.\n    train_labels[:,i] =  series_train_protein_ids.isin(label_related_proteins).astype(float)\n    \n    # Progress bar percentage increase\n    bar.update(i+1)\n\n# Notify the end of progress bar \nbar.finish()\n\n# Convert train_Y numpy into pandas dataframe\nlabels_df = pd.DataFrame(data = train_labels, columns = labels)\nprint(labels_df.shape)","metadata":{"execution":{"iopub.status.busy":"2023-07-25T08:49:50.513945Z","iopub.execute_input":"2023-07-25T08:49:50.514443Z","iopub.status.idle":"2023-07-25T08:58:28.146171Z","shell.execute_reply.started":"2023-07-25T08:49:50.514404Z","shell.execute_reply":"2023-07-25T08:58:28.145166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-25T08:58:28.147661Z","iopub.execute_input":"2023-07-25T08:58:28.148083Z","iopub.status.idle":"2023-07-25T08:58:28.179913Z","shell.execute_reply.started":"2023-07-25T08:58:28.148044Z","shell.execute_reply":"2023-07-25T08:58:28.178896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Early stopping to prevent crashing\nfrom tensorflow import keras\nfrom tensorflow.keras import layers, callbacks\nearly_stopping = callbacks.EarlyStopping(\n    min_delta = 0.005,\n    patience = 5,\n    restore_best_weights = True,\n)","metadata":{"execution":{"iopub.status.busy":"2023-07-25T08:58:28.181137Z","iopub.execute_input":"2023-07-25T08:58:28.181439Z","iopub.status.idle":"2023-07-25T08:58:28.187506Z","shell.execute_reply.started":"2023-07-25T08:58:28.181413Z","shell.execute_reply":"2023-07-25T08:58:28.186504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training (Using Tensorflow to train a Deep Neural Network with the protein \n# embeddings)\n\nINPUT_SHAPE = [train_df.shape[1]]\nBATCH_SIZE = 128\n\nmodel = tf.keras.Sequential([\n    tf.keras.layers.BatchNormalization(input_shape=INPUT_SHAPE), \n    layers.Dropout(rate = 0.3),\n    tf.keras.layers.Dense(units=512, activation='relu'),\n    tf.keras.layers.Dense(units=512, activation='relu'),\n    tf.keras.layers.Dense(units=512, activation='relu'),\n    tf.keras.layers.Dense(units=num_of_labels,activation='sigmoid')\n])\n\n\n# Compile model\nmodel.compile(\n    optimizer=tf.keras.optimizers.Adam(learning_rate=0.001),\n    loss='binary_crossentropy',\n    metrics=['binary_accuracy', tf.keras.metrics.AUC()],\n)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-25T08:58:28.188756Z","iopub.execute_input":"2023-07-25T08:58:28.189059Z","iopub.status.idle":"2023-07-25T08:58:28.462944Z","shell.execute_reply.started":"2023-07-25T08:58:28.189033Z","shell.execute_reply":"2023-07-25T08:58:28.461868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(\n    train_df, labels_df,\n    batch_size=BATCH_SIZE,\n    \n    shuffle = True,\n    \n    validation_split=0.05,\n    \n    epochs= 30,\n    callbacks = [early_stopping],\n    verbose = 0,\n)","metadata":{"execution":{"iopub.status.busy":"2023-07-25T08:58:28.464425Z","iopub.execute_input":"2023-07-25T08:58:28.464905Z","iopub.status.idle":"2023-07-25T09:05:40.345456Z","shell.execute_reply.started":"2023-07-25T08:58:28.464863Z","shell.execute_reply":"2023-07-25T09:05:40.344459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the model's loss and accuracy for each epoch\nhistory_df = pd.DataFrame(history.history)\nhistory_df.loc[:, ['loss']].plot(title=\"Cross-entropy\")\nhistory_df.loc[:, ['binary_accuracy']].plot(title=\"Accuracy\")\n","metadata":{"execution":{"iopub.status.busy":"2023-07-25T09:05:40.347956Z","iopub.execute_input":"2023-07-25T09:05:40.348387Z","iopub.status.idle":"2023-07-25T09:05:40.998025Z","shell.execute_reply.started":"2023-07-25T09:05:40.348345Z","shell.execute_reply":"2023-07-25T09:05:40.996872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(history.history.keys())","metadata":{"execution":{"iopub.status.busy":"2023-07-25T09:05:40.999539Z","iopub.execute_input":"2023-07-25T09:05:41.000398Z","iopub.status.idle":"2023-07-25T09:05:41.006163Z","shell.execute_reply.started":"2023-07-25T09:05:41.000357Z","shell.execute_reply":"2023-07-25T09:05:41.005045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_df.loc[:, ['loss', 'val_loss']].plot()","metadata":{"execution":{"iopub.status.busy":"2023-07-25T09:05:41.007652Z","iopub.execute_input":"2023-07-25T09:05:41.008065Z","iopub.status.idle":"2023-07-25T09:05:41.290153Z","shell.execute_reply.started":"2023-07-25T09:05:41.008027Z","shell.execute_reply":"2023-07-25T09:05:41.289012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Submit the protien emdeddings of the test data usuing the Rost Lab's T% protein language model\ntest_embeddings = np.load('/kaggle/input/t5embeds/test_embeds.npy')\n\n# Convert test_embeddings to dataframe\ncolumn_num = test_embeddings.shape[1]\ntest_df = pd.DataFrame(test_embeddings, columns = [\"Column_\" + str(i) for i in range(1, column_num+1)])\nprint(test_df.shape)","metadata":{"execution":{"iopub.status.busy":"2023-07-25T09:05:41.292423Z","iopub.execute_input":"2023-07-25T09:05:41.293676Z","iopub.status.idle":"2023-07-25T09:05:51.467616Z","shell.execute_reply.started":"2023-07-25T09:05:41.293632Z","shell.execute_reply":"2023-07-25T09:05:51.466653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-25T09:05:51.469047Z","iopub.execute_input":"2023-07-25T09:05:51.469513Z","iopub.status.idle":"2023-07-25T09:05:51.495012Z","shell.execute_reply.started":"2023-07-25T09:05:51.469446Z","shell.execute_reply":"2023-07-25T09:05:51.494004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Using the model to make predictions of the test\ntest_protein_ids = np.load('/kaggle/input/t5embeds/test_ids.npy')\n\npredictions =  model.predict(test_df)\ndf_submission = pd.DataFrame(predictions, index=test_protein_ids, columns=labels).stack()\ndf_submission = pd.DataFrame({'Protein Id': df_submission.index.get_level_values(0),\n                              'GO Term Id': df_submission.index.get_level_values(1),\n                              'Prediction': df_submission.values\n                             })","metadata":{"execution":{"iopub.status.busy":"2023-07-25T09:05:51.496337Z","iopub.execute_input":"2023-07-25T09:05:51.496674Z","iopub.status.idle":"2023-07-25T09:06:33.205664Z","shell.execute_reply.started":"2023-07-25T09:05:51.496646Z","shell.execute_reply":"2023-07-25T09:06:33.20405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions.shape\npredictions[1]","metadata":{"execution":{"iopub.status.busy":"2023-07-25T09:06:33.210255Z","iopub.execute_input":"2023-07-25T09:06:33.210632Z","iopub.status.idle":"2023-07-25T09:06:33.21852Z","shell.execute_reply.started":"2023-07-25T09:06:33.210599Z","shell.execute_reply":"2023-07-25T09:06:33.217376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create the submission data frame\n# Reference: https://www.kaggle.com/code/alexandervc/baseline-multilabel-to-multitarget-binary\n\ndf_submission = pd.DataFrame(columns = ['Protein Id', 'GO Term Id','Prediction'])\n","metadata":{"execution":{"iopub.status.busy":"2023-07-25T09:06:33.219989Z","iopub.execute_input":"2023-07-25T09:06:33.220445Z","iopub.status.idle":"2023-07-25T09:06:34.840478Z","shell.execute_reply.started":"2023-07-25T09:06:33.220406Z","shell.execute_reply":"2023-07-25T09:06:34.839152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l = []\nfor k in list(test_protein_ids):\n    l += [ k] * predictions.shape[1]   \n\ndf_submission['Protein Id'] = l\ndf_submission['GO Term Id'] = labels * predictions.shape[0]\ndf_submission['Prediction'] = predictions.ravel()\n#df_submission.to_csv(\"submission.tsv\",header=False, index=False, sep=\"\\t\")","metadata":{"execution":{"iopub.status.busy":"2023-07-25T09:06:34.841893Z","iopub.execute_input":"2023-07-25T09:06:34.842323Z","iopub.status.idle":"2023-07-25T09:08:06.134928Z","shell.execute_reply.started":"2023-07-25T09:06:34.84229Z","shell.execute_reply":"2023-07-25T09:08:06.133826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission","metadata":{"execution":{"iopub.status.busy":"2023-07-25T09:08:06.136158Z","iopub.execute_input":"2023-07-25T09:08:06.136461Z","iopub.status.idle":"2023-07-25T09:08:06.149962Z","shell.execute_reply.started":"2023-07-25T09:08:06.136434Z","shell.execute_reply":"2023-07-25T09:08:06.148963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nfrom sklearn.multioutput import MultiOutputClassifier\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\n\n# Split the data into training and validation sets\nX_train, X_val, y_train, y_val = train_test_split(train_df, labels_df, test_size=0.05, random_state=42)\n\n# Initialize the Random Forest Classifier\n\nrf = RandomForestClassifier(n_estimators=100, random_state=0)","metadata":{"execution":{"iopub.status.busy":"2023-07-25T10:16:00.189072Z","iopub.execute_input":"2023-07-25T10:16:00.189759Z","iopub.status.idle":"2023-07-25T10:16:01.192918Z","shell.execute_reply.started":"2023-07-25T10:16:00.189716Z","shell.execute_reply":"2023-07-25T10:16:01.191702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fit the model to the training data\nrf.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2023-07-25T10:16:14.859358Z","iopub.execute_input":"2023-07-25T10:16:14.860127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make predictions on the validation set\ny_val_pred = rf.predict(X_val)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate the accuracy of the model\naccuracy = accuracy_score(y_val, y_val_pred)\nprint(\"Validation Accuracy: \", accuracy)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install optuna","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}