{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n# Use the kagglehub client library to attach Kaggle resources like competitions, datasets, and models to your session\n# Learn more about kagglehub: https://github.com/Kaggle/kagglehub/blob/main/README.md\n\nimport kagglehub\n# kagglehub.dataset_download('<owner>/<dataset-slug>')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-06-12T16:36:36.120192Z","iopub.execute_input":"2026-06-12T16:36:36.120521Z","iopub.status.idle":"2026-06-12T16:36:37.246863Z","shell.execute_reply.started":"2026-06-12T16:36:36.12049Z","shell.execute_reply":"2026-06-12T16:36:37.245777Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"terms = pd.read_csv(\"/kaggle/input/competitions/cafa-5-protein-function-prediction/Train/train_terms.tsv\", sep = \"\\t\")\nterms.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-12T16:36:37.248049Z","iopub.execute_input":"2026-06-12T16:36:37.248563Z","iopub.status.idle":"2026-06-12T16:36:39.417044Z","shell.execute_reply.started":"2026-06-12T16:36:37.248539Z","shell.execute_reply":"2026-06-12T16:36:39.416177Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"go_terms = terms[\"term\"]\ngo_term_counts = go_terms.value_counts()\ngo_term_counts.values ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-12T16:36:39.418007Z","iopub.execute_input":"2026-06-12T16:36:39.418314Z","iopub.status.idle":"2026-06-12T16:36:39.846866Z","shell.execute_reply.started":"2026-06-12T16:36:39.418291Z","shell.execute_reply":"2026-06-12T16:36:39.846123Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"threshold = go_term_counts.quantile(0.80)\nprint(threshold)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-12T16:36:39.848713Z","iopub.execute_input":"2026-06-12T16:36:39.848915Z","iopub.status.idle":"2026-06-12T16:36:39.858268Z","shell.execute_reply.started":"2026-06-12T16:36:39.848895Z","shell.execute_reply":"2026-06-12T16:36:39.857415Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"(go_term_counts.values > 0).sum()-(go_term_counts.values < 51).sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-12T16:36:39.85961Z","iopub.execute_input":"2026-06-12T16:36:39.859979Z","iopub.status.idle":"2026-06-12T16:36:39.867682Z","shell.execute_reply.started":"2026-06-12T16:36:39.859942Z","shell.execute_reply":"2026-06-12T16:36:39.867045Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.hist(go_term_counts.values, bins=1000, edgecolor='black')\n# plt.xscale('log')\nplt.xlabel('Frequency (log scale)')\nplt.ylabel('Number of GO terms')\nplt.title('GO Term Frequency Distribution')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-12T16:36:39.868456Z","iopub.execute_input":"2026-06-12T16:36:39.868664Z","iopub.status.idle":"2026-06-12T16:36:40.932733Z","shell.execute_reply.started":"2026-06-12T16:36:39.868644Z","shell.execute_reply":"2026-06-12T16:36:40.931943Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Extracting Embedding from ESM_2 model","metadata":{}},{"cell_type":"code","source":"!kaggle datasets download -d viktorfairuschin/cafa-5-ems-2-embeddings-numpy\n!unzip -o cafa-5-ems-2-embeddings-numpy.zip\n\ntrain_ids = pd.DataFrame(np.load(\"/kaggle/working/train_ids.npy\"))\ntrain_embeddings = pd.DataFrame(np.load(\"/kaggle/working/train_embeddings.npy\"))\ntest_ids = pd.DataFrame(np.load(\"/kaggle/working/test_ids.npy\"))\ntest_embeddings = pd.DataFrame(np.load(\"/kaggle/working/test_embeddings.npy\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-12T16:36:40.933751Z","iopub.execute_input":"2026-06-12T16:36:40.933948Z","iopub.status.idle":"2026-06-12T16:37:53.539772Z","shell.execute_reply.started":"2026-06-12T16:36:40.933929Z","shell.execute_reply":"2026-06-12T16:37:53.539021Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_ids.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-12T16:37:53.541073Z","iopub.execute_input":"2026-06-12T16:37:53.541703Z","iopub.status.idle":"2026-06-12T16:37:53.547125Z","shell.execute_reply.started":"2026-06-12T16:37:53.541674Z","shell.execute_reply":"2026-06-12T16:37:53.546441Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_embeddings.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-12T16:37:53.547973Z","iopub.execute_input":"2026-06-12T16:37:53.548257Z","iopub.status.idle":"2026-06-12T16:37:53.561836Z","shell.execute_reply.started":"2026-06-12T16:37:53.548236Z","shell.execute_reply":"2026-06-12T16:37:53.560973Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# top_go_terms = np.array(go_term_counts.index[:1000]) #extracting only top 1000 go terms\n# train_terms_updated = terms.loc[terms['term'].isin(top_go_terms)] #removing all rows that have terms other then top 1000 terms\n\n\n# multihot_encoding_train_df = pd.DataFrame(columns = top_go_terms)\n# new_row = {col: 0 for col in multihot_encoding_train_df.columns}\n# new_row_df = pd.DataFrame([new_row])\n# for ID in train_ids.itertuples():\n#     multihot_encoding_train_df = pd.concat([multihot_encoding_train_df, new_row_df])\n#     for row in train_terms_updated[train_terms_updated[\"EntryID\"] == ID._1].itertuples():\n#         multihot_encoding_train_df.loc[multihot_encoding_train_df.index[-1], row.term] = 1\n        \n# multihot_encoding_train_df.head()    \n\n# 1. Filter rows to keep only the top 1000 terms\ntop_go_terms = go_term_counts.index[:1000]\ntrain_terms_updated = terms[terms['term'].isin(top_go_terms)]\n\n\n# 2. INSTANTLY build the multi-hot encoded matrix using crosstab\nmultihot_encoding_train_df = pd.crosstab(\n    index=train_terms_updated['EntryID'], \n    columns=train_terms_updated['term']\n)\n\n# 3. Ensure ALL target IDs are included (even if they have 0 terms)\n# This reindexes the rows to match your exact train_ids and fills missing spots with 0\n# (Assuming train_ids has a column or index of the true IDs)\nid_list = train_ids.iloc[:, 0].tolist() if isinstance(train_ids, pd.DataFrame) else train_ids\nmultihot_encoding_train_df = multihot_encoding_train_df.reindex(index=id_list, columns=top_go_terms, fill_value=0)\n\nmultihot_encoding_train_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-12T16:37:53.564464Z","iopub.execute_input":"2026-06-12T16:37:53.564953Z","iopub.status.idle":"2026-06-12T16:38:40.818554Z","shell.execute_reply.started":"2026-06-12T16:37:53.564931Z","shell.execute_reply":"2026-06-12T16:38:40.817856Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import tensorflow as tf\nfrom functools import partial\ninitial_bias = np.log([1 / 990]) \noutput_bias = tf.keras.initializers.Constant(initial_bias)\nsimple_hidden_layer = partial(tf.keras.layers.Dense, activation = \"relu\", kernel_initializer = \"he_normal\", kernel_regularizer = tf.keras.regularizers.l2(0.01))\nmodel = tf.keras.Sequential([\n\ttf.keras.layers.Input(shape = [1280,]),\n\tsimple_hidden_layer(1000),\n    simple_hidden_layer(1000),\n    simple_hidden_layer(1000),\n    simple_hidden_layer(1000),\n    simple_hidden_layer(1000),\n\tsimple_hidden_layer(1000, activation = \"sigmoid\", bias_initializer=output_bias)\n])\nMETRICS = [\n    tf.keras.metrics.TruePositives(name='tp'),\n    tf.keras.metrics.FalsePositives(name='fp'),\n    tf.keras.metrics.TrueNegatives(name='tn'),\n    tf.keras.metrics.FalseNegatives(name='fn'), \n    tf.keras.metrics.Precision(name='precision'),\n    tf.keras.metrics.Recall(name='recall'),\n    tf.keras.metrics.AUC(name='pr_auc', curve='PR'), # Target this for optimization\n]\n\nmodel.compile(\n    optimizer=tf.keras.optimizers.Adam(learning_rate=1e-3),\n    loss=tf.keras.losses.BinaryCrossentropy(),\n    metrics=METRICS\n)\n\nmodel.fit(train_embeddings, multihot_encoding_train_df, epochs = 2, batch_size = 64, validation_split = 0.1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-12T16:38:40.819473Z","iopub.execute_input":"2026-06-12T16:38:40.819832Z","iopub.status.idle":"2026-06-12T16:39:30.526073Z","shell.execute_reply.started":"2026-06-12T16:38:40.819809Z","shell.execute_reply":"2026-06-12T16:39:30.525331Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predictions = tf.transpose(model.predict(test_embeddings))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-12T16:39:30.527095Z","iopub.execute_input":"2026-06-12T16:39:30.527681Z","iopub.status.idle":"2026-06-12T16:39:43.553954Z","shell.execute_reply.started":"2026-06-12T16:39:30.527657Z","shell.execute_reply":"2026-06-12T16:39:43.553311Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_embeddings.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-12T16:39:43.555005Z","iopub.execute_input":"2026-06-12T16:39:43.555298Z","iopub.status.idle":"2026-06-12T16:39:43.56059Z","shell.execute_reply.started":"2026-06-12T16:39:43.555268Z","shell.execute_reply":"2026-06-12T16:39:43.559775Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_submission = pd.DataFrame(columns = ['Protein Id', 'GO Term Id','Prediction'])\nl = []\nfor k in test_ids.itertuples():\n    l += [ k._1] * len(top_go_terms)   \n    \ndf_submission['Protein Id'] = l\ndf_submission['GO Term Id'] = top_go_terms.tolist() * predictions.shape[-1]\ndf_submission['Prediction'] = tf.reshape(predictions, [-1])\ndf_submission.to_csv(\"submission.tsv\",header=False, index=False, sep=\"\\t\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-12T16:47:18.620801Z","iopub.execute_input":"2026-06-12T16:47:18.621594Z","iopub.status.idle":"2026-06-12T16:54:22.286221Z","shell.execute_reply.started":"2026-06-12T16:47:18.621566Z","shell.execute_reply":"2026-06-12T16:54:22.285572Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nprint(os.path.abspath(\"submission.tsv\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-12T16:54:22.287512Z","iopub.execute_input":"2026-06-12T16:54:22.287842Z","iopub.status.idle":"2026-06-12T16:54:22.292174Z","shell.execute_reply.started":"2026-06-12T16:54:22.287819Z","shell.execute_reply":"2026-06-12T16:54:22.291527Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 1. Install from the correct repo\n!pip install git+https://github.com/claradepaolis/CAFA-evaluator-PK.git -q\n\n# 2. Verify -toi and -known now appear\n!cafaeval --help","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-12T16:54:22.293154Z","iopub.execute_input":"2026-06-12T16:54:22.293701Z","iopub.status.idle":"2026-06-12T16:54:34.647335Z","shell.execute_reply.started":"2026-06-12T16:54:22.293672Z","shell.execute_reply":"2026-06-12T16:54:34.646595Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\n# cafaeval expects a directory, not a file\nos.makedirs('/kaggle/working/predictions/', exist_ok=True)\n!cp /kaggle/input/notebooks/siddhvr/cafa5-ems2-embeds-with-pytorch/submission.tsv /kaggle/working/predictions/\n\n\nobo='/kaggle/input/datasets/sanjayashrestha123/cafa-5-eval/go-basic.obo'\nmethod_dir='/kaggle/working/predictions' # specify directory with predictions to evaluate\ngt='/kaggle/input/datasets/sanjayashrestha123/cafa-5-eval/eval_terms_no_knowledge_2025_03.tsv'\nia='/kaggle/input/datasets/sanjayashrestha123/cafa-5-eval/IA_t0.tsv'\ntoi='/kaggle/input/datasets/sanjayashrestha123/cafa-5-eval/toi_2025_03.tsv'\nknown='/kaggle/input/datasets/sanjayashrestha123/cafa-5-eval/known_t0.tsv'\n\n!cafaeval {obo} {method_dir} {gt} -ia {ia} -toi {toi} -known {known} -prop fill -th_step 0.001 -no_orphans","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-12T16:54:34.649576Z","iopub.execute_input":"2026-06-12T16:54:34.650198Z","iopub.status.idle":"2026-06-12T17:05:19.946174Z","shell.execute_reply.started":"2026-06-12T16:54:34.650168Z","shell.execute_reply":"2026-06-12T17:05:19.945397Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\ndf_all  = pd.read_csv('/kaggle/working/results/evaluation_all.tsv', sep='\\t')\ndf_best_f = pd.read_csv('/kaggle/working/results/evaluation_best_f.tsv', sep='\\t')\n\nprint(df_best_f.to_string())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-12T17:05:19.947525Z","iopub.execute_input":"2026-06-12T17:05:19.947898Z","iopub.status.idle":"2026-06-12T17:05:19.980395Z","shell.execute_reply.started":"2026-06-12T17:05:19.947857Z","shell.execute_reply":"2026-06-12T17:05:19.979791Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import auc\n\ndf = pd.read_csv('/kaggle/working/results/evaluation_all.tsv', sep='\\t')\nprint(df.columns.tolist())\nprint(df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-12T17:05:19.98123Z","iopub.execute_input":"2026-06-12T17:05:19.981569Z","iopub.status.idle":"2026-06-12T17:05:20.120223Z","shell.execute_reply.started":"2026-06-12T17:05:19.981547Z","shell.execute_reply":"2026-06-12T17:05:20.119357Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 3, figsize=(15, 5))\n\nnamespaces = {\n    'biological_process': 'BP',\n    'cellular_component': 'CC', \n    'molecular_function': 'MF'\n}\n\ncolors = {'biological_process': 'blue', \n          'cellular_component': 'green', \n          'molecular_function': 'red'}\n\nfor ax, (ns, ns_short) in zip(axes, namespaces.items()):\n    subset = df[(df['ns'] == ns) & (df['filename'] == 'submission.tsv')].copy()\n    subset = subset.sort_values('rc')  # sort by recall for clean curve\n    \n    pr = subset['pr'].values\n    rc = subset['rc'].values\n    fmax = subset['f'].max()\n    pr_auc = auc(rc, pr)\n    \n    ax.plot(rc, pr, color=colors[ns], linewidth=2, \n            label=f'AUC={pr_auc:.3f}, Fmax={fmax:.3f}')\n    \n    # Mark the Fmax point\n    best_row = subset.loc[subset['f'].idxmax()]\n    ax.scatter(best_row['rc'], best_row['pr'], \n               color='black', zorder=5, s=80, label=f'Fmax point (τ={best_row[\"tau\"]:.3f})')\n    \n    ax.set_xlabel('Recall', fontsize=12)\n    ax.set_ylabel('Precision', fontsize=12)\n    ax.set_title(f'{ns_short} Precision-Recall Curve', fontsize=13)\n    ax.legend(fontsize=10)\n    ax.set_xlim([0, 1])\n    ax.set_ylim([0, 1])\n    ax.grid(alpha=0.3)\n\nplt.suptitle('ESM-2 Model — PR Curves per Namespace', fontsize=14, fontweight='bold')\nplt.tight_layout()\nplt.savefig('/kaggle/working/pr_curves.png', dpi=150, bbox_inches='tight')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-12T17:05:20.121822Z","iopub.execute_input":"2026-06-12T17:05:20.122057Z","iopub.status.idle":"2026-06-12T17:05:21.095026Z","shell.execute_reply.started":"2026-06-12T17:05:20.122034Z","shell.execute_reply":"2026-06-12T17:05:21.09427Z"}},"outputs":[],"execution_count":null}]}