{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Implicit recommender baseline\n* NOTE: This does not include inference, auxiliary features or the other fun stuff (including target processing) - I leave that out here :)\n* Uses tensorflow recommenders - retrieval task. \n* can compare to other approaches","metadata":{"id":"adb27795"}},{"cell_type":"code","source":"!pip install tensorflow_recommenders  -q -U","metadata":{"id":"5Oz4oOw-E7YA","scrolled":true,"execution":{"iopub.status.busy":"2023-04-23T11:57:31.099423Z","iopub.execute_input":"2023-04-23T11:57:31.100099Z","iopub.status.idle":"2023-04-23T11:57:54.12337Z","shell.execute_reply.started":"2023-04-23T11:57:31.100065Z","shell.execute_reply":"2023-04-23T11:57:54.122132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\n    \nfrom sklearn.feature_extraction.text import CountVectorizer\n# import scipy.sparse as sp\nfrom numpy import log2\nfrom tensorflow.keras.utils import Sequence\nfrom scipy.spatial.distance import cosine\n\nimport tensorflow as tf\nfrom keras.layers import Input, Embedding, Dot, Flatten, Concatenate, Dense\nfrom keras.models import Model\nfrom keras.optimizers import Adam\nfrom sklearn.model_selection import train_test_split\n# from sentence_transformers import SentenceTransformer\nimport os\nimport pprint\nimport tempfile\nfrom typing import Dict, Text\nimport tensorflow_recommenders as tfrs\ntf.get_logger().setLevel('FATAL') #ERROR\n\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import mixed_precision\n# %matplotlib inline","metadata":{"id":"80fa0a67","execution":{"iopub.status.busy":"2023-04-23T11:57:54.126094Z","iopub.execute_input":"2023-04-23T11:57:54.126499Z","iopub.status.idle":"2023-04-23T11:58:00.608087Z","shell.execute_reply.started":"2023-04-23T11:57:54.126455Z","shell.execute_reply":"2023-04-23T11:58:00.60694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_indexed_annot_preds(prot_index_num):\n    _, titles = index(tf.constant([str(prot_index_num)]))\n    # print(f\"Recommendations for {prot_index_num}, {user_ids[prot_index_num]},: {titles[0, :10]}\")\n    print(f\"Recommendations for {prot_index_num}, {user_ids[prot_index_num]}\")\n    titles = list(titles[0, :].numpy())\n    titles = [i.decode('utf8') for i in titles]\n    # annotations that protein already ahs: \n    existing_annots = data.loc[data[\"user_idx\"]==prot_index_num][\"item_id\"].unique()\n\n    # print(\"existing_annots\",existing_annots)\n    print(\"Novel predictions (not) existing annots):\",[i for i in titles if i not in existing_annots])\n    print(\"predictions present in existing annots:\",[i for i in titles if i in existing_annots])","metadata":{"execution":{"iopub.status.busy":"2023-04-23T11:58:00.60975Z","iopub.execute_input":"2023-04-23T11:58:00.610499Z","iopub.status.idle":"2023-04-23T11:58:00.62148Z","shell.execute_reply.started":"2023-04-23T11:58:00.610466Z","shell.execute_reply":"2023-04-23T11:58:00.620498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2023-04-23T11:58:00.626145Z","iopub.execute_input":"2023-04-23T11:58:00.626436Z","iopub.status.idle":"2023-04-23T11:58:00.645682Z","shell.execute_reply.started":"2023-04-23T11:58:00.62641Z","shell.execute_reply":"2023-04-23T11:58:00.644759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))","metadata":{"id":"GSc9xUuiUek_","outputId":"8ef2fba6-a2c8-43b2-b12d-8a6880e322a5","execution":{"iopub.status.busy":"2023-04-23T11:58:00.64696Z","iopub.execute_input":"2023-04-23T11:58:00.64739Z","iopub.status.idle":"2023-04-23T11:58:00.868332Z","shell.execute_reply.started":"2023-04-23T11:58:00.647355Z","shell.execute_reply":"2023-04-23T11:58:00.867256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# USE_PRETRAINED_DL = False\nFAST_RUN = False#True\n\nGET_k_TOP_PREDS = 250\n\nif FAST_RUN:\n    GET_k_TOP_PREDS = 20","metadata":{"id":"7dec0a98","execution":{"iopub.status.busy":"2023-04-23T11:58:00.870074Z","iopub.execute_input":"2023-04-23T11:58:00.870466Z","iopub.status.idle":"2023-04-23T11:58:00.879801Z","shell.execute_reply.started":"2023-04-23T11:58:00.870427Z","shell.execute_reply":"2023-04-23T11:58:00.878764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_file =\"/kaggle/input/cafa-5-protein-function-prediction/Train/train_terms.tsv\"","metadata":{"execution":{"iopub.status.busy":"2023-04-23T11:58:00.88325Z","iopub.execute_input":"2023-04-23T11:58:00.883542Z","iopub.status.idle":"2023-04-23T11:58:00.891159Z","shell.execute_reply.started":"2023-04-23T11:58:00.883516Z","shell.execute_reply":"2023-04-23T11:58:00.890252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.read_csv(data_file,sep=\"\\t\",usecols=[\"EntryID\",\"term\"]).drop_duplicates()\nprint(data.shape[0])\ndata.columns = ['user_id','item_id'] \nif FAST_RUN:\n    data = data.sample(150_000)\n    it_cnts = data['item_id'].value_counts()\n    data = data[data['item_id'].map(it_cnts) > 100]\n    data = data[data['user_id'].map(data['user_id'].value_counts()) > 3]\n    print(data.shape[0],\"# rows after fast run filter\")\n\n#     ## keep sample of mosre popular annotations for speedup\nit_cnts = data['item_id'].value_counts()\ndata = data[data['item_id'].map(it_cnts) > 1]\nprint(data.nunique())\nprint(data[\"user_id\"].value_counts().describe().round(1))\nprint(data[\"item_id\"].value_counts())\ndisplay(data)","metadata":{"id":"e500c6a3","outputId":"a5daaf0b-e356-4b2f-8ca2-a7d10421d6bb","execution":{"iopub.status.busy":"2023-04-23T11:58:00.89291Z","iopub.execute_input":"2023-04-23T11:58:00.89364Z","iopub.status.idle":"2023-04-23T11:58:06.139046Z","shell.execute_reply.started":"2023-04-23T11:58:00.893604Z","shell.execute_reply":"2023-04-23T11:58:06.13801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"user_ids = data['user_id'].unique()\nitem_ids = data['item_id'].unique()\n\nuser_idx_map = {user_id: idx for idx, user_id in enumerate(user_ids)}\nitem_idx_map = {item_id: idx for idx, item_id in enumerate(item_ids)}","metadata":{"id":"3e6eca8d","execution":{"iopub.status.busy":"2023-04-23T11:58:06.140605Z","iopub.execute_input":"2023-04-23T11:58:06.141713Z","iopub.status.idle":"2023-04-23T11:58:06.150954Z","shell.execute_reply.started":"2023-04-23T11:58:06.141674Z","shell.execute_reply":"2023-04-23T11:58:06.149989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['item_id'] = data['item_id'].astype(str)\ndata['user_id'] = data['user_id'].astype(str) ","metadata":{"id":"f72c3c2c","execution":{"iopub.status.busy":"2023-04-23T11:58:06.155736Z","iopub.execute_input":"2023-04-23T11:58:06.156044Z","iopub.status.idle":"2023-04-23T11:58:06.170937Z","shell.execute_reply.started":"2023-04-23T11:58:06.156018Z","shell.execute_reply":"2023-04-23T11:58:06.16999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['user_idx'] = data['user_id'].map(user_idx_map)\ndata['item_idx'] = data['item_id'].map(item_idx_map)\n\n## add:\nuser_idxs = data['user_idx'].unique()\nitem_idxs = data['item_idx'].unique()\n\nunique_user_ids = data['user_id'].unique()\nunique_movie_titles = data['item_id'].unique()","metadata":{"id":"b80929aa","execution":{"iopub.status.busy":"2023-04-23T11:58:06.172338Z","iopub.execute_input":"2023-04-23T11:58:06.172847Z","iopub.status.idle":"2023-04-23T11:58:06.191627Z","shell.execute_reply.started":"2023-04-23T11:58:06.172809Z","shell.execute_reply":"2023-04-23T11:58:06.190673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CF model","metadata":{"id":"60dd6541"}},{"cell_type":"markdown","source":"### adapt TFRS \n* https://www.kaggle.com/code/mfaaris/hybrid-and-tensorflow-recommender-system#4\n* note diff - user vs movie ids etc\n","metadata":{"id":"c3c12606"}},{"cell_type":"code","source":"movies_df = data[[\"item_id\",\"item_idx\"]].drop_duplicates().reset_index(drop=True)\nmovies_df","metadata":{"id":"44fd9bb3","outputId":"1bdca442-1a05-42e4-e877-dac5d4e9f4fe","execution":{"iopub.status.busy":"2023-04-23T11:58:06.19332Z","iopub.execute_input":"2023-04-23T11:58:06.193871Z","iopub.status.idle":"2023-04-23T11:58:06.211813Z","shell.execute_reply.started":"2023-04-23T11:58:06.193834Z","shell.execute_reply":"2023-04-23T11:58:06.210547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Basic Implicit recommendation retrieval model\n###### https://stackoverflow.com/questions/66750232/usage-of-factorizedtopk-in-tensorflow-recommenders\n* Dot product. In batch negatives\n* https://www.tensorflow.org/recommenders/examples/basic_retrieval\n\n* Note suggested base Learning Rate is Crazy high - tutorials used 0.5~0.1 for adagrad! ","metadata":{"id":"6048a0a3"}},{"cell_type":"code","source":"# Variables\nseed = 42\ntest_percentage = 10\ntrain_percentage = 100-test_percentage\nembedding_dimension =  192#32 \nmetrics_batchsize = 256#16\ntrain_batchsize = 256#1024#128\ntest_batchsize = 256#1024#64\nlearning_rate = 0.1#0.004 # 0.5 ?\nepochs = 15#3\nindex_batchsize = GET_k_TOP_PREDS\n\n\nviews_df =  data[[\"user_id\",\"item_id\"]]\nusers_df = data[\"user_id\"].unique()\ncontents_df = views_df['item_id'].unique()\n\nviews_ds = tf.data.Dataset.from_tensor_slices(dict(views_df))\ncontents_ds = tf.data.Dataset.from_tensor_slices(contents_df)","metadata":{"id":"34939b83","execution":{"iopub.status.busy":"2023-04-23T11:58:06.213476Z","iopub.execute_input":"2023-04-23T11:58:06.21384Z","iopub.status.idle":"2023-04-23T11:58:08.498075Z","shell.execute_reply.started":"2023-04-23T11:58:06.213802Z","shell.execute_reply":"2023-04-23T11:58:08.497052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"view_size = len(views_df)\ntrain_size = round(view_size/100*train_percentage)\ntest_size = view_size-train_size\n\ntf.random.set_seed(seed)\n\nviews_ds_shuffled = views_ds.shuffle(len(views_df), seed=seed, reshuffle_each_iteration=False)\ntrain = views_ds_shuffled.take(train_size)\ntest = views_ds_shuffled.skip(train_size).take(test_size)","metadata":{"id":"5ae305a9","execution":{"iopub.status.busy":"2023-04-23T11:58:08.499458Z","iopub.execute_input":"2023-04-23T11:58:08.499896Z","iopub.status.idle":"2023-04-23T11:58:08.534064Z","shell.execute_reply.started":"2023-04-23T11:58:08.499859Z","shell.execute_reply":"2023-04-23T11:58:08.533127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"user_model = tf.keras.Sequential([\ntf.keras.layers.StringLookup(vocabulary=users_df, mask_token=None),\ntf.keras.layers.Embedding(input_dim=len(users_df) + 1, output_dim=embedding_dimension)\n])\n\ncontent_model = tf.keras.Sequential([\ntf.keras.layers.StringLookup(\n    vocabulary=unique_movie_titles, mask_token=None),\ntf.keras.layers.Embedding(input_dim=len(contents_df) + 1, output_dim=embedding_dimension)])\n    \ncandidates=contents_ds.batch(metrics_batchsize).map(content_model)\n\nmetrics = tfrs.metrics.FactorizedTopK(\n  candidates=candidates,\n    ks = (1, 10,50, 100#,300 # warning - slow\n          )\n)\n\ntask = tfrs.tasks.Retrieval(\n  metrics=metrics\n#   ,remove_accidental_hits = True # ValueError: When accidental hit removal is enabled, candidate ids must be supplied.\n)\n\nclass ContentModel(tfrs.Model):\n\n  def __init__(self, user_model, content_model):\n    super().__init__()\n    self.content_model: tf.keras.Model = content_model\n    self.user_model: tf.keras.Model = user_model\n    self.task: tf.keras.layers.Layer = task\n\n  def compute_loss(self, features: Dict[Text, tf.Tensor], training=False) -> tf.Tensor:\n    content_embeddings = self.content_model(features[\"item_id\"])\n    user_embeddings = self.user_model(features[\"user_id\"])\n    return self.task(query_embeddings=user_embeddings,\n                        candidate_embeddings=content_embeddings,\n                    # candidate_ids=contents_df\n                     ## speed up train:\n                     compute_metrics=not training, # speed up training , lose outputs\n                    )\n","metadata":{"id":"9b3eeb2a","execution":{"iopub.status.busy":"2023-04-23T11:58:08.535753Z","iopub.execute_input":"2023-04-23T11:58:08.536499Z","iopub.status.idle":"2023-04-23T11:58:08.861745Z","shell.execute_reply.started":"2023-04-23T11:58:08.536461Z","shell.execute_reply":"2023-04-23T11:58:08.860745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmodel = ContentModel(user_model, content_model)\n\nmodel.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate))\n# model.compile(optimizer=tf.keras.optimizers.Adagrad(learning_rate=0.1))\n\ncached_train = train.shuffle(view_size).batch(train_batchsize).cache()\ncached_test = test.batch(8192).cache()","metadata":{"id":"843a769f","outputId":"6582d230-3cd2-4e0d-d133-2f2231a48d6c","execution":{"iopub.status.busy":"2023-04-23T11:58:08.863196Z","iopub.execute_input":"2023-04-23T11:58:08.86355Z","iopub.status.idle":"2023-04-23T11:58:08.896301Z","shell.execute_reply.started":"2023-04-23T11:58:08.863508Z","shell.execute_reply":"2023-04-23T11:58:08.895364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ncallback = tf.keras.callbacks.EarlyStopping(patience=2)\n\nmodel.fit(cached_train, epochs=epochs, \n          validation_data = cached_test\n          , callbacks=[callback],\n          )\nprint(\"test\")\nmodel.evaluate(cached_test, return_dict=True)","metadata":{"id":"6b78e6c8","outputId":"6fbd532c-e4f7-482a-f29c-1aa1dcba6945","scrolled":true,"execution":{"iopub.status.busy":"2023-04-23T11:58:08.898688Z","iopub.execute_input":"2023-04-23T11:58:08.899438Z","iopub.status.idle":"2023-04-23T11:58:19.644482Z","shell.execute_reply.started":"2023-04-23T11:58:08.899399Z","shell.execute_reply":"2023-04-23T11:58:19.643349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Index - use to get predictions / candidates\n* Can submit these or rerank using a larger downstream (retreival/reranker) style model\n* NOTE: test data is not just cold start, but needs to be provided for model. \n    * Will be done elsewhere as part of target augmentation (See ProteinBERT, Linial lab entry in CAFA 3+4). \n    \n* ToDo: map human readable GO descriptions instead of IDs. ","metadata":{}},{"cell_type":"code","source":"%%time\n# Create a model that takes in raw query features, and\nindex = tfrs.layers.factorized_top_k.BruteForce(query_model=model.user_model,k=GET_k_TOP_PREDS)\n# recommends movies out of the entire (TRAIN)dataset.\nindex.index_from_dataset(\ntf.data.Dataset.zip((contents_ds.batch(GET_k_TOP_PREDS), contents_ds.batch(GET_k_TOP_PREDS).map(model.content_model)))\n)","metadata":{"id":"lkmSay7MSUVD","outputId":"30c2a6f3-cabc-4010-fcda-62d30481889e","execution":{"iopub.status.busy":"2023-04-23T11:58:19.646233Z","iopub.execute_input":"2023-04-23T11:58:19.646626Z","iopub.status.idle":"2023-04-23T11:58:19.718833Z","shell.execute_reply.started":"2023-04-23T11:58:19.646587Z","shell.execute_reply":"2023-04-23T11:58:19.71763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### View some predictions ","metadata":{}},{"cell_type":"code","source":"get_indexed_annot_preds(1)","metadata":{"id":"0cde9f42","outputId":"26f9c15e-2fd2-4075-8554-49df75d6e608","execution":{"iopub.status.busy":"2023-04-23T11:58:19.720182Z","iopub.execute_input":"2023-04-23T11:58:19.720833Z","iopub.status.idle":"2023-04-23T11:58:19.748226Z","shell.execute_reply.started":"2023-04-23T11:58:19.720793Z","shell.execute_reply":"2023-04-23T11:58:19.747155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### To get submissions - use index over test set. After adding data there and handling the cold start :)\n* I leave that as an exercise to the reader , or the upcoming paper I scratched this code out from!","metadata":{}},{"cell_type":"code","source":"## this should actually be on test set seqs (submission seems to include too many sequences) - parse IDs from fastas or taxonomy\ndata_sub = pd.read_csv(\"/kaggle/input/cafa-5-protein-function-prediction/sample_submission.tsv\",sep=\"\\t\",header=None).iloc[:,0:1].drop_duplicates()\nsub_ds = tf.data.Dataset.from_tensor_slices(data_sub)\ndisplay(data_sub)","metadata":{"execution":{"iopub.status.busy":"2023-04-23T11:58:19.749799Z","iopub.execute_input":"2023-04-23T11:58:19.75018Z","iopub.status.idle":"2023-04-23T11:58:20.042229Z","shell.execute_reply.started":"2023-04-23T11:58:19.750143Z","shell.execute_reply":"2023-04-23T11:58:20.041248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### BruteForce predictions retrieval: \n* This'll only work properly once having a model that can handle cold start, need to format index correctly\n","metadata":{}},{"cell_type":"code","source":"%%time\nlist_id = []\npred_labels = []\npred_score = []\n\nscores, title = index.predict([data_sub.values])\n\ntitle = title.flatten()\ntitle = [i.decode('utf8') for i in title]\nlist_id = []\nfor row in data_sub.itertuples(index=False):\n    list_id.extend(list(row)*GET_k_TOP_PREDS)","metadata":{"execution":{"iopub.status.busy":"2023-04-23T11:58:20.043503Z","iopub.execute_input":"2023-04-23T11:58:20.044175Z","iopub.status.idle":"2023-04-23T11:58:34.878943Z","shell.execute_reply.started":"2023-04-23T11:58:20.044131Z","shell.execute_reply":"2023-04-23T11:58:34.877812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submit = pd.DataFrame({\"EntryID\":list_id,\n              \"term\":title,\n              \"Score\":scores.flatten()}).round(3)\ndisplay(df_submit)","metadata":{"execution":{"iopub.status.busy":"2023-04-23T11:58:34.880633Z","iopub.execute_input":"2023-04-23T11:58:34.881029Z","iopub.status.idle":"2023-04-23T11:58:35.362203Z","shell.execute_reply.started":"2023-04-23T11:58:34.880992Z","shell.execute_reply":"2023-04-23T11:58:35.360078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submit.to_csv(\"submission.tsv\",header=False,index=False,sep=\"\\t\")","metadata":{"execution":{"iopub.status.busy":"2023-04-23T11:58:35.363637Z","iopub.execute_input":"2023-04-23T11:58:35.364266Z","iopub.status.idle":"2023-04-23T11:58:39.136032Z","shell.execute_reply.started":"2023-04-23T11:58:35.364227Z","shell.execute_reply":"2023-04-23T11:58:39.135007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub2 = df_submit.sort_values([\"EntryID\",\"Score\"],ascending=False)\ndf_sub2 = df_sub2.groupby(\"EntryID\").head(n = (GET_k_TOP_PREDS//3)).reset_index()\ndisplay(df_sub2)\ndf_sub2.to_csv(\"submission2.tsv\",header=False,index=False,sep=\"\\t\")","metadata":{},"execution_count":null,"outputs":[]}]}