{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Install Torchmetrics and Torchsummary packages","metadata":{}},{"cell_type":"code","source":"%%capture \n!pip install torchmetrics","metadata":{"execution":{"iopub.status.busy":"2023-05-30T03:56:02.585451Z","iopub.execute_input":"2023-05-30T03:56:02.585828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n!pip install torchsummary","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import necessary packages","metadata":{}},{"cell_type":"code","source":"## https://www.kaggle.com/code/alexandervc/baseline-multilabel-to-multitarget-binary#Load-train-features---precalculated-embeddings-for-the-proteins\nimport os\nimport gc\nfrom sklearn.model_selection import train_test_split\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm\ntqdm.pandas()\nimport torch\nimport warnings\nwarnings.filterwarnings('ignore')\nfrom sklearn.model_selection import train_test_split\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchmetrics import AUROC,F1Score\nfrom torchmetrics.classification import BinaryF1Score\nfrom torchsummary import summary as torchsummary\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataframe functions\n","metadata":{}},{"cell_type":"code","source":"def get_dataframe(path):\n    return pd.read_csv(path,sep='\\t')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_terms = '/kaggle/input/cafa-5-protein-function-prediction/Train/train_terms.tsv'\ntrain_taxonomy ='/kaggle/input/cafa-5-protein-function-prediction/Train/train_taxonomy.tsv'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_dataframe(train_terms).head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_dataframe(train_taxonomy).head()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Summary Function","metadata":{}},{"cell_type":"code","source":"def summary(text, df):\n    print(f'{text} shape: {df.shape}')\n    summ = pd.DataFrame(df.dtypes, columns=['dtypes'])\n    summ['null'] = df.isnull().sum()\n    summ['unique'] = df.nunique()\n    summ['min'] = df.min()\n    summ['median'] = df.median()\n    summ['max'] = df.max()\n    summ['mean'] = df.mean()\n    summ['std'] = df.std()\n    summ['duplicate'] = df.duplicated().sum()\n    return summ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reduce memory function","metadata":{}},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Summaries","metadata":{}},{"cell_type":"code","source":"summary('train_terms',reduce_mem_usage(get_dataframe(train_terms)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summary('train_terms',reduce_mem_usage(get_dataframe(train_taxonomy)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Countplot for various aspects","metadata":{}},{"cell_type":"code","source":"sns.countplot(data=reduce_mem_usage(get_dataframe(train_terms)),x='aspect',color='r')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_terms=reduce_mem_usage(get_dataframe(train_terms))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_dataset():\n    train_protein_ids = np.load('/kaggle/input/t5embeds/train_ids.npy')\n    train_embeddings = np.load('/kaggle/input/t5embeds/train_embeds.npy')\n    column_num = train_embeddings.shape[1]\n    train = pd.DataFrame(train_embeddings, columns = [\"Column_\" + str(i) for i in range(1, column_num+1)])\n    return train,train_protein_ids\n\ntrain,train_protein_ids = get_train_dataset()\nprint(train.shape,train_protein_ids.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_of_labels = 1500\ndef get_label_train_terms(df):\n    labels=df['term'].value_counts().index[:num_of_labels].tolist()\n    train_terms_updated=df.loc[df['term'].isin(labels)]\n    return labels,train_terms_updated\n\nlabels_count,train_terms_updated=get_label_train_terms(train_terms)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_pit_aspects():\n    pie_df = train_terms_updated['aspect'].value_counts()\n    palette_color = sns.color_palette('pastel')\n    plt.pie(pie_df.values, labels=np.array(pie_df.index), colors=palette_color, autopct='%.0f%%')\n    plt.show()\n    \nshow_pit_aspects()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_labels(train_protein_ids):\n    train_size = train_protein_ids.shape[0] # len(X)\n    train_labels = np.zeros((train_size ,num_of_labels))\n    series_train_protein_ids = pd.Series(train_protein_ids)\n\n    for i in range(num_of_labels):\n        n_train_terms = train_terms_updated[train_terms_updated['term'] ==  labels_count[i]]\n        label_related_proteins = n_train_terms['EntryID'].unique()\n        train_labels[:,i] =  series_train_protein_ids.isin(label_related_proteins).astype(float)\n    return train_labels\n\n\ntrain_labels=get_labels(train_protein_ids)\n\nlabels = pd.DataFrame(data = train_labels, columns = labels_count)\nprint(labels.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_test_dataset(features,labels):\n    return  train_test_split(features,labels,shuffle=True,random_state=42)\n\nX_train,X_val,y_train,y_val = train_test_dataset(train,labels)\nprint(X_train.shape,X_val.shape,y_train.shape,y_val.shape)\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nprint(device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FluxData(Dataset):\n    def __init__(self, X_data, y_data):\n        self.X_data = X_data\n        self.y_data = y_data\n        \n    def __getitem__(self, index):\n            return self.X_data[index], self.y_data[index]\n        \n    def __len__ (self):\n        return len(self.X_data)\n\nX_data = torch.from_numpy(X_train.values).float().to(device)\ny_data = torch.from_numpy(y_train.values).float().to(device)\nX_val = torch.from_numpy(X_val.values).float().to(device)\ny_val = torch.from_numpy(y_val.values).float().to(device)\ntrain_data = FluxData(X_data,y_data)\ntest_data = FluxData(X_val,y_val)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CAFA5NNetBase(torch.nn.Module):\n    \n    def training_step(self,batch):\n        features,labels = batch\n        out = self(features)\n        loss = F.binary_cross_entropy(out,labels)\n        return loss\n    \n    def validation_step(self, batch):\n        features, labels = batch \n        out = self(features)                    # Generate predictions\n        loss = F.binary_cross_entropy(out, labels)   # Calculate loss\n        acc = auroc(out, labels)           # Calculate accuracy\n        return {'Validation_loss': loss.detach(), 'Validation_acc': acc}\n        \n    def validation_epoch_end(self, outputs):\n        batch_losses = [x['Validation_loss'] for x in outputs]\n        epoch_loss = torch.stack(batch_losses).mean()   # Combine losses\n        batch_accs = [x['Validation_acc'] for x in outputs]\n        epoch_acc = torch.stack(batch_accs).mean()      # Combine accuracies\n        return {'Validation_loss': epoch_loss.item(), 'Validation_acc': epoch_acc.item()}\n    \n    def epoch_end(self, epoch, result):\n        if epoch%5==0:\n            print(\"Epoch [{}], Train_loss: {:.4f}, Validation_loss: {:.4f}, Validation_acc: {:.4f}\".format(\n            epoch, result['Train_loss'], result['Validation_loss'], result['Validation_acc']))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CAFA5NNet(CAFA5NNetBase):\n    def __init__(self,input_features,output_features):\n        super(CAFA5NNet,self).__init__()\n        self.network = torch.nn.Sequential(\n        torch.nn.Linear(input_features,1024),\n        torch.nn.ReLU(),\n        torch.nn.Linear(1024,1024),    \n        torch.nn.ReLU(),\n        torch.nn.Linear(1024,output_features),\n        torch.nn.Sigmoid()\n        )\n    def forward(self,inputs):\n        return self.network(inputs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = CAFA5NNet(X_train.shape[1],y_train.shape[1])\nmodel.to(device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torchsummary(model, X_data.size(), batch_size=-1, device='cuda')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 5120\nEPOCHS = 50\nLEARNING_RATE = 0.001\nMOMENTUM = 0.9\nOPT_FUNC = torch.optim.Adam","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_dataloaders(dataset_type,batch,shuffle):\n    if shuffle:\n         return DataLoader(dataset=dataset_type, batch_size=batch, shuffle=True)\n    else:\n        return DataLoader(dataset=dataset_type, batch_size=batch,shuffle=False)\n    \ntrain_dl = get_dataloaders(train_data,BATCH_SIZE,True)\nval_dl = get_dataloaders(test_data,BATCH_SIZE,False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def auroc(outputs, labels):\n    auroc = AUROC(task=\"binary\")\n    return auroc(outputs, labels)\n\n  \n@torch.no_grad()\ndef evaluate(model, val_loader):\n    model.eval()\n    outputs = [model.validation_step(batch) for batch in val_loader]\n    return model.validation_epoch_end(outputs)\n\n  \ndef fit(epochs, lr, model, train_loader, val_loader, opt_func = OPT_FUNC):\n    \n    history = []\n    optimizer = opt_func(model.parameters(),lr)\n    for epoch in tqdm(range(epochs)):\n        \n        model.train()\n        train_losses = []\n        for batch in train_loader:\n            loss = model.training_step(batch)\n            train_losses.append(loss)\n            loss.backward()\n            optimizer.step()\n            optimizer.zero_grad()\n            \n        result = evaluate(model, val_loader)\n        result['Train_loss'] = torch.stack(train_losses).mean().item()\n        model.epoch_end(epoch, result)\n        history.append(result)\n    \n    return history","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = fit(EPOCHS, LEARNING_RATE, model, train_dl, val_dl,OPT_FUNC)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_accuracies(history):\n    \"\"\" Plot the history of accuracies\"\"\"\n    accuracies = [x['Validation_acc'] for x in history]\n    plt.plot(accuracies, '-x')\n    plt.xlabel('Epoch')\n    plt.ylabel('Accuracy')\n    plt.title('Accuracy vs. No. of epochs');\n    \n\nplot_accuracies(history)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_losses(history):\n    \"\"\" Plot the losses in each epoch\"\"\"\n    train_losses = [x.get('Train_loss') for x in history]\n    val_losses = [x['Validation_loss'] for x in history]\n    plt.plot(train_losses, '-bx')\n    plt.plot(val_losses, '-rx')\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.legend(['Training', 'Validation'])\n    plt.title('Loss vs. No. of epochs');\n\nplot_losses(history)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_test_dataset():\n    test_embeddings = np.load('/kaggle/input/t5embeds/test_embeds.npy')\n    column_num = test_embeddings.shape[1]\n    test = pd.DataFrame(test_embeddings, columns = [\"Column_\" + str(i) for i in range(1, column_num+1)])\n    return test \n\ntest = get_test_dataset()\nprint(test.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CAFA5TestData(Dataset):\n    \n    def __init__(self, X_test_data):\n        self.X_test_data = X_test_data\n        \n    def __getitem__(self, index):\n        return self.X_test_data[index]\n        \n    def __len__ (self):\n        return len(self.X_test_data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = CAFA5TestData(torch.from_numpy(test.values).float().to(device))\ntest_data_loader = DataLoader(dataset=test_data, batch_size=test.shape[0])\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef eval_test_data(model,testing_data_dl):\n    model.eval()\n    with torch.no_grad():\n        for X_batch_test in testing_data_dl:\n            X_batch_test = X_batch_test.to(device)\n            predictions = model(X_batch_test)\n            prediction_target=predictions.detach().numpy()\n\n    return prediction_target\n\nprediction_target = eval_test_data(model,test_data_loader)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction_target.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_predictions(prediction_target):\n    test_protein_ids = np.load('/kaggle/input/t5embeds/test_ids.npy')\n    protein_list = []\n    for k in list(test_protein_ids):\n        protein_list += [k] * prediction_target.shape[1] \n    return protein_list\n\nprotein_list=make_predictions(prediction_target)      ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def submit():\n    df_submission = pd.DataFrame(columns = ['Protein Id', 'GO Term Id','Prediction'])\n    df_submission['Protein Id'] = protein_list\n    df_submission['GO Term Id'] = labels_count * prediction_target.shape[0]\n    df_submission['Prediction'] = prediction_target.ravel()\n    df_submission.to_csv(\"submission.tsv\",header=False, index=False,sep='\\t')\n    \nsubmit()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp=pd.read_csv('/kaggle/working/submission.tsv',sep='\\t')\ntemp.count()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}