{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":41875,"databundleVersionId":5521661,"sourceType":"competition"},{"sourceId":5607816,"sourceType":"datasetVersion","datasetId":3225525},{"sourceId":6216319,"sourceType":"datasetVersion","datasetId":3569938},{"sourceId":6221861,"sourceType":"datasetVersion","datasetId":3573385},{"sourceId":6222023,"sourceType":"datasetVersion","datasetId":3573469},{"sourceId":6222633,"sourceType":"datasetVersion","datasetId":3573825},{"sourceId":6244802,"sourceType":"datasetVersion","datasetId":3523659},{"sourceId":6245049,"sourceType":"datasetVersion","datasetId":3588444},{"sourceId":6245839,"sourceType":"datasetVersion","datasetId":3588867},{"sourceId":6249897,"sourceType":"datasetVersion","datasetId":3591647},{"sourceId":6249948,"sourceType":"datasetVersion","datasetId":3591675},{"sourceId":6250004,"sourceType":"datasetVersion","datasetId":3591707},{"sourceId":6250098,"sourceType":"datasetVersion","datasetId":3591759},{"sourceId":6250187,"sourceType":"datasetVersion","datasetId":3591815},{"sourceId":6250347,"sourceType":"datasetVersion","datasetId":3591902},{"sourceId":6259593,"sourceType":"datasetVersion","datasetId":3597735},{"sourceId":6259750,"sourceType":"datasetVersion","datasetId":3597827},{"sourceId":6260035,"sourceType":"datasetVersion","datasetId":3597993},{"sourceId":6278630,"sourceType":"datasetVersion","datasetId":3609819},{"sourceId":6279113,"sourceType":"datasetVersion","datasetId":3610121},{"sourceId":6279337,"sourceType":"datasetVersion","datasetId":3610244},{"sourceId":6279769,"sourceType":"datasetVersion","datasetId":3610546},{"sourceId":6289275,"sourceType":"datasetVersion","datasetId":3616886},{"sourceId":6289936,"sourceType":"datasetVersion","datasetId":3617278},{"sourceId":6290805,"sourceType":"datasetVersion","datasetId":3617855}],"dockerImageVersionId":30497,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Imports and Configurations**","metadata":{"_uuid":"663ae8a2-e827-4215-a5d2-521d720a5602","_cell_guid":"3a8fd238-2acc-4420-9920-a8441d8e5784","trusted":true}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n\nimport torch\n#from Bio import SeqIO\n#import plotly.graph_objects as go\n#from collections import Counter\n\nimport sys\n\n#from transformers import BertModel, BertTokenizer\nimport re\n\nfrom timeit import default_timer as timer\nfrom datetime import timedelta\nimport time \nfrom datetime import datetime\n\nfrom glob import glob\n\nfrom scipy.special import softmax\n\n\nfrom torchmetrics.classification import BinaryF1Score\n\nnp.set_printoptions(precision=5)\n\ntorch.set_printoptions(threshold=5,sci_mode=False)\ntorch.manual_seed(0)","metadata":{"_uuid":"0e9f5ca9-8724-442a-8ee6-a26375764a11","_cell_guid":"abd8cb38-f08f-477d-9e72-a5614ce986b4","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#os.remove(\"/kaggle/working/training_data.csv\")","metadata":{"_uuid":"67648565-97e6-4259-99fd-8583d05a846b","_cell_guid":"a5a3528e-e4cb-45f0-8400-1b8576895254","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-08-12T09:11:40.547632Z","iopub.execute_input":"2023-08-12T09:11:40.547903Z","iopub.status.idle":"2023-08-12T09:11:40.553013Z","shell.execute_reply.started":"2023-08-12T09:11:40.54788Z","shell.execute_reply":"2023-08-12T09:11:40.551913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ndevice","metadata":{"_uuid":"59be3fcb-e408-4999-91fa-39493e03fdf9","_cell_guid":"028a943b-29c1-46e9-bc6e-b4fcef05419b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-08-12T09:11:40.554607Z","iopub.execute_input":"2023-08-12T09:11:40.55504Z","iopub.status.idle":"2023-08-12T09:11:40.593037Z","shell.execute_reply.started":"2023-08-12T09:11:40.555002Z","shell.execute_reply":"2023-08-12T09:11:40.592105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_data = np.load(\"/kaggle/input/2500-go/2500_go.npz\", allow_pickle=True)","metadata":{"execution":{"iopub.status.busy":"2023-08-12T09:11:40.596061Z","iopub.execute_input":"2023-08-12T09:11:40.596704Z","iopub.status.idle":"2023-08-12T09:11:40.625236Z","shell.execute_reply.started":"2023-08-12T09:11:40.596672Z","shell.execute_reply":"2023-08-12T09:11:40.624298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EasyFeedForward(torch.nn.Module):\n    \n        def __init__(self, input_size, hidden_size, output_size):\n            super(EasyFeedForward, self).__init__()\n            self.layer1 = torch.nn.Linear(input_size, hidden_size)\n            self.activiation1 = torch.nn.ReLU()\n            self.dropout1 = torch.nn.Dropout(0.5)\n            self.layer2 = torch.nn.Linear(hidden_size, hidden_size)\n            self.activiation2 = torch.nn.ReLU()\n            self.dropout2 = torch.nn.Dropout(0.5)\n            self.layer3 = torch.nn.Linear(hidden_size, hidden_size)\n            self.activiation3 = torch.nn.ReLU()\n            self.layer4 = torch.nn.Linear(hidden_size, hidden_size)\n            self.activiation4 = torch.nn.ReLU()\n            self.layer5 = torch.nn.Linear(hidden_size, hidden_size)\n            self.activiation5 = torch.nn.ReLU()\n            self.layer6 = torch.nn.Linear(hidden_size, hidden_size)\n            self.activiation6 = torch.nn.ReLU()\n            self.layer7 = torch.nn.Linear(hidden_size, output_size)\n            \n        def forward(self, x):\n            output = self.layer1(x)\n            output = self.activiation1(output)\n            output = self.dropout1(output)\n            output = self.layer2(output)\n            output = self.activiation2(output)\n            output = self.dropout2(output)\n            output = self.layer3(output)\n            output = self.activiation3(output)\n            output = self.layer4(output)\n            output = self.activiation4(output)\n            output = self.layer5(output)\n            output = self.activiation5(output)\n            output = self.layer6(output)\n            output = self.activiation6(output)\n            output = self.layer7(output)\n            return output\n\n#(1) init FeedForwardNet \nnn_input_size  = 1024 #last_hidden from ProtBert\nnn_hidden_size = 2500\nnn_output_size = 2500 #500        \n        \nff_nn_model = EasyFeedForward(nn_input_size,nn_hidden_size,nn_output_size ).to(device)\n\n# print(\"NeuralNet uses gpu/tpu: \" + str(next(ff_nn_model.parameters()).is_cuda))\n# print(\"\")\n# ff_nn_model.parameters","metadata":{"_uuid":"36f348ee-17e7-4138-92ea-881b1b8f5429","_cell_guid":"fd8b9ed7-1148-414c-be30-42f119cac0ee","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-08-12T09:11:40.626939Z","iopub.execute_input":"2023-08-12T09:11:40.627568Z","iopub.status.idle":"2023-08-12T09:11:46.182944Z","shell.execute_reply.started":"2023-08-12T09:11:40.627536Z","shell.execute_reply":"2023-08-12T09:11:46.181984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ff_nn_model","metadata":{"execution":{"iopub.status.busy":"2023-08-12T09:11:46.184668Z","iopub.execute_input":"2023-08-12T09:11:46.185368Z","iopub.status.idle":"2023-08-12T09:11:46.192184Z","shell.execute_reply.started":"2023-08-12T09:11:46.185335Z","shell.execute_reply":"2023-08-12T09:11:46.191269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"listoflhst = []\nlistoftargetVecs = []\n\n\nfor i, emb in enumerate(training_data[\"last_hidden_state\"]):\n    if type(emb).__module__ == \"numpy\":\n        listoflhst.append(torch.from_numpy(emb))\n    elif type(emb).__module__ == \"torch\":\n        listoflhst.append(emb)\n    else:\n        print(type(emb).__module__, \"unsupported datatype\")\n        \n        \nfor i, vec in enumerate(training_data[\"target_vec\"]):\n    if type(emb).__module__ == \"numpy\":\n        listoftargetVecs.append(torch.from_numpy(vec))\n    elif type(emb).__module__ == \"torch\":\n        listoftargetVecs.append(vec[0])\n    else:\n        print(type(emb).__module__, \"unsupported datatype\")\n\n        \nlhstVecs = torch.stack(listoflhst, 0)\n\nprint(len(lhstVecs))\n\ntargetVecs = torch.stack(listoftargetVecs, 0)\ntargetVecs = targetVecs.float()\nprint(len(targetVecs))","metadata":{"_uuid":"1e0d9c39-02cd-491f-85f0-f858d10deef1","_cell_guid":"47b2f8df-1a1b-48a2-892f-c00b981d3747","execution":{"iopub.status.busy":"2023-08-12T09:11:46.193895Z","iopub.execute_input":"2023-08-12T09:11:46.194582Z","iopub.status.idle":"2023-08-12T09:13:00.329008Z","shell.execute_reply.started":"2023-08-12T09:11:46.19455Z","shell.execute_reply":"2023-08-12T09:13:00.326917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"shuffle","metadata":{}},{"cell_type":"code","source":"indexes_shuffle = torch.randperm(len(targetVecs))\n\n\nlhstVecs   = lhstVecs[indexes_shuffle]\ntargetVecs = targetVecs[indexes_shuffle]\n","metadata":{"execution":{"iopub.status.busy":"2023-08-12T09:13:00.330599Z","iopub.execute_input":"2023-08-12T09:13:00.331041Z","iopub.status.idle":"2023-08-12T09:13:01.686945Z","shell.execute_reply.started":"2023-08-12T09:13:00.331008Z","shell.execute_reply":"2023-08-12T09:13:01.685953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lhstVecsGPU = lhstVecs.to(device)\ntargetVecsGPU = targetVecs.to(device)\n#print(lhstVecsGPU, targetVecsGPU)\nprint(len(lhstVecsGPU), len(targetVecsGPU))","metadata":{"_uuid":"07526ade-ce1d-4a99-891e-c60e55c41e70","_cell_guid":"e32a7302-1172-4205-b7fa-278bfa884a46","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-08-12T09:13:01.688551Z","iopub.execute_input":"2023-08-12T09:13:01.693917Z","iopub.status.idle":"2023-08-12T09:13:02.102694Z","shell.execute_reply.started":"2023-08-12T09:13:01.693881Z","shell.execute_reply":"2023-08-12T09:13:02.101654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#split train test\nsplit_num = 50_000\ntrain_lhstVecsGPU, test_lhstVecsGPU     = lhstVecsGPU[:split_num], lhstVecsGPU[split_num:len(lhstVecsGPU)] \ntrain_targetVecsGPU, test_targetVecsGPU = targetVecsGPU[:split_num], targetVecsGPU[split_num:len(targetVecsGPU)] ","metadata":{"execution":{"iopub.status.busy":"2023-08-12T09:13:02.106124Z","iopub.execute_input":"2023-08-12T09:13:02.106412Z","iopub.status.idle":"2023-08-12T09:13:02.117577Z","shell.execute_reply.started":"2023-08-12T09:13:02.106386Z","shell.execute_reply":"2023-08-12T09:13:02.116284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_targetVecsGPU)","metadata":{"execution":{"iopub.status.busy":"2023-08-12T09:13:02.118939Z","iopub.execute_input":"2023-08-12T09:13:02.119353Z","iopub.status.idle":"2023-08-12T09:13:02.132798Z","shell.execute_reply.started":"2023-08-12T09:13:02.119321Z","shell.execute_reply":"2023-08-12T09:13:02.131757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#from datetime import datetime\nprint(\"start\")\nmetric = BinaryF1Score(multidim_average='global',threshold=0.1)\nmetric.to(device)\n\ncrossEntropyLoss = torch.nn.BCEWithLogitsLoss(reduction=\"mean\").to(device)\n\noptimizer = torch.optim.Adam(ff_nn_model.parameters(), lr=0.0001)\nn_epochs = 30\n\nbatch_size = 16\nprint(\"is cuda: \", next(ff_nn_model.parameters()).is_cuda)\n\nprint(\"BEGIN TRAINING...\", datetime.now())\nstart = timer()\nfor epoch in range(n_epochs):\n    \n    \n    for i in range(0,len(train_targetVecsGPU), batch_size):\n        lhst = train_lhstVecsGPU[i:i+batch_size]\n        target = train_targetVecsGPU[i:i+batch_size]\n        optimizer.zero_grad()\n        out = ff_nn_model(lhst)\n        loss = crossEntropyLoss(out,target)\n        loss.backward()\n        optimizer.step()\n    if epoch % 1 == 0:\n        \n        #f1 score\n        out = ff_nn_model(test_lhstVecsGPU)  \n        f1_score = metric(out,test_targetVecsGPU)\n  \n        \n        end = timer()\n        print(epoch, loss.detach().item(),\"f1: \", f1_score.detach().cpu().item(),timedelta(seconds=end-start),datetime.now())\n        start = timer()\n        \n        \n          \n","metadata":{"_uuid":"34c4b2c5-a182-454d-bc15-cd25c8263f54","_cell_guid":"17ad449e-71ce-4ce3-b5bd-146c9aad64cc","execution":{"iopub.status.busy":"2023-08-12T09:40:20.336894Z","iopub.execute_input":"2023-08-12T09:40:20.337917Z","iopub.status.idle":"2023-08-12T09:46:00.191819Z","shell.execute_reply.started":"2023-08-12T09:40:20.337883Z","shell.execute_reply":"2023-08-12T09:46:00.188238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ff_nn_model_loaded = torch.load(\"new_model_1000_bce_loss_deeper_with_f1_score_and_dropout_2\",map_location=torch.device('cpu'))\nff_nn_model_loaded","metadata":{"execution":{"iopub.status.busy":"2023-08-12T09:46:22.69714Z","iopub.execute_input":"2023-08-12T09:46:22.697602Z","iopub.status.idle":"2023-08-12T09:46:22.810983Z","shell.execute_reply.started":"2023-08-12T09:46:22.697564Z","shell.execute_reply":"2023-08-12T09:46:22.810044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.nn.functional.sigmoid(ff_nn_model_loaded(lhstVecs[num ]))","metadata":{"execution":{"iopub.status.busy":"2023-08-12T09:47:05.857462Z","iopub.execute_input":"2023-08-12T09:47:05.857819Z","iopub.status.idle":"2023-08-12T09:47:05.882941Z","shell.execute_reply.started":"2023-08-12T09:47:05.857792Z","shell.execute_reply":"2023-08-12T09:47:05.881887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, t , tv  in zip(range(len(torch.nn.functional.sigmoid(ff_nn_model_loaded(lhstVecs[num ])))),torch.nn.functional.sigmoid(ff_nn_model_loaded(lhstVecs[num ])),targetVecs[num ] ) :\n    #print(t)\n    #if tv.item() == 1 or tv.item() == 0:\n    if t.item() > 0.7: \n        print(i, tv,t.item())","metadata":{"execution":{"iopub.status.busy":"2023-08-12T09:49:40.839016Z","iopub.execute_input":"2023-08-12T09:49:40.839376Z","iopub.status.idle":"2023-08-12T09:49:40.887649Z","shell.execute_reply.started":"2023-08-12T09:49:40.839347Z","shell.execute_reply":"2023-08-12T09:49:40.886684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num = 0\n\nt_batch = ff_nn_model(lhstVecs[num ].to(device))\nt_batch = torch.nn.functional.sigmoid(t_batch)\n\ntensi = t_batch.detach().cpu()\ntype(list(tensi.numpy()))\n\nfor i, t , tv  in zip(range(len(tensi)),tensi,targetVecs[num ] ) :\n    #print(t)\n    #if tv.item() == 1 or tv.item() == 0:\n    if t.item() > 0.7: \n        print(i, tv,t.item())","metadata":{"execution":{"iopub.status.busy":"2023-08-12T09:48:40.205242Z","iopub.execute_input":"2023-08-12T09:48:40.205609Z","iopub.status.idle":"2023-08-12T09:48:40.230595Z","shell.execute_reply.started":"2023-08-12T09:48:40.205582Z","shell.execute_reply":"2023-08-12T09:48:40.229552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(ff_nn_model, \"new_model_1000_bce_loss_deeper_with_f1_score_and_dropout_2\")","metadata":{"execution":{"iopub.status.busy":"2023-08-12T09:34:02.951129Z","iopub.execute_input":"2023-08-12T09:34:02.951495Z","iopub.status.idle":"2023-08-12T09:34:03.194969Z","shell.execute_reply.started":"2023-08-12T09:34:02.951468Z","shell.execute_reply":"2023-08-12T09:34:03.194016Z"},"trusted":true},"execution_count":null,"outputs":[]}]}