{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# What is about ?\n\n\nModeling notebook with blending different models and datas \n\n","metadata":{"papermill":{"duration":0.016009,"end_time":"2023-07-15T21:44:45.8759","exception":false,"start_time":"2023-07-15T21:44:45.859891","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Key params","metadata":{}},{"cell_type":"code","source":"list_main_configs_models_etc = []\n# epochs = 15\n# batch_size = 128\n\n# Current tops : \n\n# 679 L2 = 3*2000+2*2200 , DR=0.35 ---------------- results seems a little better \n# blend5\n# 0.9259\n# 0.465295\n\n# 667 droupout1 = 0.4 -------------------- even better results, but so stable \n# blend5\n# 0.9278\n# 0.46494\n\n# 691 L3 also -------------------------------- 1e-6,1e-8,1e-8 - \n# blend5\n# 0.9267\n# 0.464887\n\n\n# 675 L2 = 2200 , DR=0.35 ------------------------------------------------ top params but not so stable compere 676\n# blend5\n# 0.9267\n# 0.464477\n\n# 666 droupout1 = 0.35 rerun ----------- results are stable surprsingly \n# blend5\n# 0.9267\n# 0.463797\n\n# 657 baseline again small mode 5*KMLP1 rerun --------------- initial baseline \n# blend5\n# 0.9252\n# 0.461914\n# compare 653 \n# blend5\n# 0.9257\n# 0.462257\n\n\ndroupout_loc = 0.35; droupout_loc2 = 0.3; \ncfg = {'Model': ['KMLP', {'Layers': [1000,2000,None,None], 'Dropouts':[droupout_loc,droupout_loc2,droupout_loc,droupout_loc] , \n 'BatchNormalizations':[True,False,False,False] , 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \nlist_main_configs_models_etc += [cfg ]*1\ncfg = {'Model': ['KMLP', {'Layers': [1000,2500,None,None], 'Dropouts':[droupout_loc,droupout_loc2,droupout_loc,droupout_loc] , \n 'BatchNormalizations':[True,False,False,False] , 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \nlist_main_configs_models_etc += [cfg ]*1\ncfg = {'Model': ['KMLP', {'Layers': [1500,2000,None,None], 'Dropouts':[droupout_loc,droupout_loc2,droupout_loc,droupout_loc] , \n 'BatchNormalizations':[True,False,False,False] , 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \nlist_main_configs_models_etc += [cfg ]*1\ncfg = {'Model': ['KMLP', {'Layers': [1500,2300,None,None], 'Dropouts':[droupout_loc,droupout_loc2,droupout_loc,droupout_loc] , \n 'BatchNormalizations':[True,False,False,False] , 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \nlist_main_configs_models_etc += [cfg ]*1\n\ndroupout_loc = 0.4; droupout_loc2 = 0.3; \ncfg = {'Model': ['KMLP', {'Layers': [1000,2300,None,None], 'Dropouts':[droupout_loc,droupout_loc2,droupout_loc,droupout_loc] , \n 'BatchNormalizations':[True,False,False,False] , 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \nlist_main_configs_models_etc += [cfg ]*1\n\n# 735 5configs around basic - wider\n# 734 5configs around basic - wider\n\n# 733 5configs around basic\n# 732 5configs around basic\n\n# 731 basic config rerun - L2 = 0 all \n# 730 basic config rerun - L2 = 0 all \n\n# 729 model.add(blend5\n# blend5\n# 0.9265\n# 0.458541\n\n# 728 model.add(LayerNormalization()) 0,0,1e-8, \n# blend5\n# 0.926\n# 0.456998\n\n# 727 model.add(LayerNormalization()) 0,0,1e-8, minmaxscaler on all TSVD t5\n# blend5\n# 0.9246\n# 0.454417\n\n# 726 model.add(LayerNormalization()) 0,0,1e-8, minmaxscaler on all TSVD t5\n# blend5\n# 0.9229\n# 0.454788\n\n\n# 724 minmaxscaler on all TSVD t5 conccat l2 0,0,1e-7 L2 = 2000 rerun\n# blend5\n# 0.9228\n# 0.453986\n\n# 723 minmaxscaler on all TSVD t5 conccat l2 0,0,1e-7 L2 = 2000\n# blend5\n# 0.9222\n# 0.456321\n\n\n# 722 t5 minmaxscaler TSVD conccat l2 0,0,1e-7 L2 = 2000 rerun\n# blend5\n# 0.9266\n# 0.463339\n\n# 721 t5 minmaxscaler TSVD conccat l2 0,0,1e-7 L2 = 2000\n# blend5\n# 0.9262\n# 0.461448\n\n\n# 720 t5 TSVD conccat l2 0,0,1e-7 L2 = 2000 rerun\n# blend5\n# 0.9267\n# 0.462203\n\n# 719 t5 TSVD conccat l2 0,0,1e-7 L2 = 2000\n# blend5\n# 0.9255\n# 0.46381\n\n# 718 l2 0,0,1e-7 L2 = 2000 rerun\n# blend5\n# 0.9263\n# 0.462702\n\n# 717 l2 0,0,1e-7 L2 = 2000\n# blend5\n# 0.9265\n# 0.46259\n\n\n# 716 l2 0,0,1e-7 L2 = 2200 rerun\n# blend5\n# 0.9268\n# 0.462808\n\n# 715 l2 0,0,1e-7 L2 = 2200\n# blend5\n# 0.9265\n# 0.463234\n\n\n# and Repeat another current best candidate \n# 714 Dr0.35,0.3 1000 2200 1e-6,1e-8,1e-8 rerun \n# blend5\n# 0.9261\n# 0.464351\n\n# 713 Dr0.35,0.3 1000 2200 1e-6,1e-8,1e-8 \n# blend5\n# 0.926\n# 0.463232\n\n# 712 - bug\n\n# Repeat current best candidate \n# 711 Dr0.35,0.3 1000 2000 1e-6,1e-8,1e-8 rerun ---------------- problem with reproducibilty (((((((((())))))))))\n# blend5\n# 0.9261\n# 0.461618\n\n# 710 Dr0.35,0.3 1000 2000 1e-6,1e-8,1e-8 \n# blend5\n# 0.9263\n# 0.463719\n\n# 709 Dr0.35,0.2 1000 2000 1e-6,1e-8,1e-8 rerun \n# blend5\n# 0.9263\n# 0.461016\n\n# 708 Dr0.35,0.2 1000 2000 1e-6,1e-8,1e-8 \n# blend5\n# 0.9267\n# 0.463129\n\n\n# 707 Dr0.35,0.25 1000 2000 1e-6,1e-8,1e-8 rerun \n# blend5\n# 0.9269\n# 0.462269\n\n# 706 Dr0.35,0.25 1000 2000 1e-6,1e-8,1e-8 \n\n# 705 Dr0.35,0.4 1000 2000 1e-6,1e-8,1e-8 rerun \n# blend5\n# 0.9262\n# 0.462835\n\n# 704 Dr0.35,0.4 1000 2000 1e-6,1e-8,1e-8 \n# blend5\n# 0.9266\n# 0.459695\n\n\n# 703 Dr0.35,0.35 1000 2000 1e-6,1e-8,1e-8 rerun \n# blend5\n# 0.9269\n# 0.46377\n\n# 702 Dr0.35,0.35 1000 2000 1e-6,1e-8,1e-8 \n# blend5\n# 0.9272\n# 0.461356\n\n# 701 Dr0.35 1200 2500 1e-6,1e-8,1e-8 - rerun \n# blend5\n# 0.9263\n# 0.462074\n\n\n# 700 Dr0.35 1200 2500 1e-6,1e-8,1e-8 \n# blend5\n# 0.9256\n# 0.4624\n\n# 699 Dr0.35 1200 2200 1e-6,1e-8,1e-8 \n# blend5\n# 0.9259\n# 0.461547\n\n# 698 Dr0.35 1200 2200 1e-6,1e-8,1e-8 \n# blend5\n# 0.9266\n# 0.462337\n\n\n# 697 Dr0.35 2200 1e-6,1e-8,1e-8 - rerun ----------- stangely bad results \n# blend5\n# 0.9268\n# 0.460815\n\n# 696 Dr0.35 2200 1e-6,1e-8,1e-8 -\n# blend5\n# 0.9261\n# 0.463093\n\n# 695 Dr0.4 2200 1e-6,1e-8,1e-8 - rerun \n# blend5\n# 0.9248\n# 0.46204\n\n# 694 Dr0.4 2200 1e-6,1e-8,1e-8 - \n# blend5\n# 0.9271\n# 0.463718\n\n# 693 ---------------- 1e-6 1e-8 1e-9\n# blend5\n# 0.9265\n# 0.463033\n\n# 691 L3 also -------------------------------- 1e-6,1e-8,1e-8 - \n# blend5\n# 0.9267\n# 0.464887\n\n# 690 \n# blend5\n# 0.9263\n# 0.46233\n\n# 689\n# blend5\n# 0.926\n# 0.462548\n\n# 688 \n# blend5\n# 0.9271\n# 0.463169\n\n# 687 \n# blend5\n# 0.9266\n# 0.461442\n\n# 686 \n# blend5\n# 0.9268\n# 0.463611\n\n\n# 682 basic DR0.35, kernel_regularizer=keras.regularizers.l2(1e-6) \n# blend5\n# 0.9262\n# 0.462997\n\n\n# 681 basic DR0.35, kernel_regularizer=keras.regularizers.l2(1e-8) \n# blend5\n# 0.9259\n# 0.463371\n\n\n# 679 L2 = 3*2000+2*2000 , DR=0.35 ---------------- results seems a little better \n# blend5\n# 0.9259\n# 0.465295\n\n\n# 678 L2 = 2500 , DR=0.4\n# blend5\n# 0.926\n# 0.463418\n\n# 677 L2 = 2500 , DR=0.35\n# blend5\n# 0.9263\n# 0.463423\n\n\n# 676 L2 = 2200 , DR=0.35\n# blend5\n# 0.9261\n# 0.463344\n\n# 675 L2 = 2200 , DR=0.35 ------------------------------------------------ top params but not so stable compere 676\n# blend5\n# 0.9267\n# 0.464477\n\n# 674 L2 = 1500 , DR=0.35\n# blend5\n# 0.9283\n# 0.462445\n\n\n# 673 L2 = 1500 , DR=0.35\n# blend5\n# 0.9272\n# 0.46233\n\n# 672 L1 = 800 , DR=0.35\n# blend5\n# 0.9276\n# 0.462844\n\n\n# 671 L1 = 1200 , DR=0.35\n# blend5\n# 0.9262\n# 0.463238\n\n# 670 L1 = 1500 , DR=0.35\n# blend5\n# 0.9256\n# 0.462532\n\n# 669 droupout1 = 0.5\n# blend5\n# 0.9269\n# 0.461852\n\n# 668 droupout1 = 0.4 rerun\n# blend5\n# 0.9278\n# 0.463286\n\n# 667 droupout1 = 0.4 -------------------- even better results, but so stable \n# blend5\n# 0.9278\n# 0.46494\n\n# 666 droupout1 = 0.35 rerun ----------- results are stable surprsingly \n# blend5\n# 0.9267\n# 0.463797\n\n# 665 droupout1 = 0.25\n# blend5\n# 0.9261\n# 0.461009\n\n\n# 664 droupout1 = 0.35 ------------------- WOW -------------- !!!!!!!!!!!!!!!!!! improvement over basic \n# blend5\n# 0.9269\n# 0.463791\n\n# 663 swish - 1,2 layer\n# blend5\n# 0.9239\n# 0.459989\n\n\n# 662 swish - 2 layer\n# blend5\n# 0.9253\n# 0.460006\n\n\n# 661 swish - 1 layer\n# blend5\n# 0.9246\n# 0.460032\n\n# 660 'epochs' : 10\n# blend5\n# 0.928\n# 0.461512\n\n\n# 659 'epochs' : 16\n# blend5\n# 0.9254\n# 0.461977\n\n# 658 'epochs' : 13\n# blend5\n# 0.9269\n# 0.461654\n\n\n# 657 baseline again small mode 5*KMLP1 rerun \n# blend5\n# 0.9252\n# 0.461914\n# compare 653 \n# blend5\n# 0.9257\n# 0.462257\n\n\n# 656 SELU first layer small mode 5*KMLP1 ----------------------- worse than baseline 653 \n# blend5\n# 0.9223\n# 0.461773\n\n# 655 tanh small mode 5*KMLP1 - last layer\n# Very bad 0.333\n\n# 654 softmax small mode 5*KMLP1 rerun \n# model.add(Dense(nlabels, activation='softmax'))\n# F1 is zero - may be due to threshold , auc is Ok\n\n# 653 small mode 5*KMLP1 rerun ----------------------------------------------------- baseline \n# blend5\n# 0.9257\n# 0.462257\n\n\n# # # # # previous best - but better in blend \n# # # # skMLP1*5\n# cfg = {'Model': ['skMLP', {'Layers': [500,1000] } ] } \n# list_main_configs_models_etc += [cfg ]*1\n\n# # Best: skMLP_MI500_RS12_HL[1500, 2000]_alpha0.1 C11 0.8658 0.458203\n# cfg = {'Model': ['skMLP', {'Layers': [1500,2000] , 'alpha':0.1 } ] } # , 'Features': dict_data_cfg1 } \n# list_main_configs_models_etc += [cfg ]*1\n\n\n# # with skip connections \nreg1 = 1e-8\n# for a1 in ['relu','swish','selu']: # activation\n# for a2 in ['relu','swish','selu']: # activation\n# for reg1 in [1e-8]:\n# for LS1 in [2000]:\n# for LS2 in [2000]:\n# cfg = {'Model': ['KMLP2', {'Layers': [LS1,LS2] , 'reg_l2':[reg1,reg1,reg1,reg1,reg1], 'activation':['selu','relu'] , 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \n# list_main_configs_models_etc += [cfg ]*1\n\n# str_optimizer = model_cfg[1].get('optimizer', 'adam')\n# learning_rate = model_cfg[1].get('LR', 0.001 ) \n \n# reg1 = 1e-7; cfg = {'Model': ['KMLP2', {'Layers': [2000,2000],'LR':0.0001,'optimizer':'adam', 'reg_l2':[reg1,reg1,reg1,reg1,reg1], 'activation':['selu','relu'] , 'epochs' : 30 , 'batch_size' : 128 , 'verbose' : 0 } ] } \n# list_main_configs_models_etc += [cfg ]*1\n\nLS = 2000; \n\n# for reg1a in [1e-8,1e-7,1e-6]: \n# for reg1b in [1e-8,1e-7,1e-6]: \n# for reg1c in [1e-8,1e-7,1e-6]: \n# for reg1d in [1e-8,1e-7,1e-6]: \n# for reg1e in [1e-8,1e-7,1e-6]: \n# LS1,LS2,LS3,LS4 = 2000,1000,1000,2000\n# for LS1 in [1000,1500,2000]:\n# for LS2 in [500,1000,1500, 2000]:\n# for LS3 in [500,1000,1500, 2000]:\n# for LS4 in [1000,1500, 2000]:\n# LS1,LS2,LS3,LS4 = 2000,1000,1000,2000\n# reg1a,reg1b,reg1c,reg1d,reg1e = 1e-7,1e-7,1e-7,1e-7,1e-7 \n# cfg = {'Model': ['KMLP3', {'Layers': [LS1,LS2,LS3,LS4],'LR':0.001,'optimizer':'adam', 'reg_l2':[reg1a,reg1b,reg1c,reg1d,reg1e], 'activation':['selu','relu','relu','relu'] , \n# 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \n# list_main_configs_models_etc += [cfg ]*10\n\n","metadata":{"execution":{"iopub.status.busy":"2023-07-20T20:34:53.829794Z","iopub.execute_input":"2023-07-20T20:34:53.830304Z","iopub.status.idle":"2023-07-20T20:34:53.855005Z","shell.execute_reply.started":"2023-07-20T20:34:53.830263Z","shell.execute_reply":"2023-07-20T20:34:53.853132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# work_mode = 'bigModeForSubmit' # 'smallModeToExplore_1'\nwork_mode = 'smallModeToExplore_1'\n\nif work_mode == 'bigModeForSubmit':\n\n n_labels_to_consider = 1850# 100\n mode_Y_source_files_to_use = 'Y1850' # 'Y31466' # \n list_features = ['T5_43k', 'esm2Fairushchin_43k' , 'taxons31_onehot_43k' ] # , 'F06_genes' # ['T5_43k']# \n n_folds_to_process = 100 # For speed-up we can compute only on say 1 or 2 ... folds \n str_cv_info_default = 'cv_folds_5'\n n_samples_to_consider = 43189 # 10_000# 10_000 # Mainly for debug purposes - we can reduce number of samples to run faster\n\n cutoff_threshold_low = 0.1\n\n\n # Scoring can be quite slow , so use scoring of each model with care \n flag_scoring_each_model = False # True # False # True # False # \n flag_scoring_each_blend = False # True # True # False # \n flag_scoring_final = True # True # False # True # False # \n\n mode_submit = True # False # None\n \nelif work_mode == 'smallModeToExplore_1':\n \n n_labels_to_consider = 1850# 100\n mode_Y_source_files_to_use = 'Y1850' # 'Y31466' # \n list_features = ['T5_43k', 'esm2Fairushchin_43k' , 'taxons31_onehot_43k' ] # , 'F06_genes' # ['T5_43k']# \n n_folds_to_process = 1 # For speed-up we can compute only on say 1 or 2 ... folds \n str_cv_info_default = 'cv_folds_2'\n n_samples_to_consider = 10_000# 43189 # 10_000 # Mainly for debug purposes - we can reduce number of samples to run faster\n cutoff_threshold_low = 0.1\n # Scoring can be quite slow , so use scoring of each model with care \n flag_scoring_each_model = True # True # False # True # False # \n flag_scoring_each_blend = True # True # True # False # \n flag_scoring_final = False # True # False # True # False # \n mode_submit = None # False # None\n \nflag_correct_metric_computation_bug_found_by_Anton = True\nblend_mode = 'mean'\n ","metadata":{"papermill":{"duration":0.034459,"end_time":"2023-07-15T21:44:45.949824","exception":false,"start_time":"2023-07-15T21:44:45.915365","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-20T20:34:53.965567Z","iopub.execute_input":"2023-07-20T20:34:53.966007Z","iopub.status.idle":"2023-07-20T20:34:53.979355Z","shell.execute_reply.started":"2023-07-20T20:34:53.965969Z","shell.execute_reply":"2023-07-20T20:34:53.978008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Main configuration setting for models, etc ","metadata":{"papermill":{"duration":0.01295,"end_time":"2023-07-15T21:44:45.975782","exception":false,"start_time":"2023-07-15T21:44:45.962832","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# dict_data_cfg10 = {'list_features_ids': [ 'E5120_43k'] }\n# dict_data_cfg11 = {'list_features_ids': ['T5_43k', 'E5120_43k', 'taxons31_onehot_43k' ] }\n\n# cfg0 = {'Model': ['Ridge',{'alpha':100} ], 'Features': dict_data_cfg11 } # , 'n_folds': 3 }\n# list_main_configs_models_etc = [cfg0,cfg0,cfg0,cfg0,cfg0]\n\n# cfg = {'Model': ['skMLP', { } ], 'Features': dict_data_cfg10b } \n# list_main_configs_models_etc.append(cfg) \n\n# dict_data_cfg1 = {'list_features_ids': ['T5_43k', 'esm2Fairushchin_43k' , 'taxons31_onehot_43k' ] } \n# list_main_configs_models_etc = []\n# # for alpha in [1e-1, 1e-2, 1e-3, 1e-4,1e-5]:\n# for learning_rate_init in [1e-1, 1e-2, 1e-3, 1e-4,1e-5]:\n# for L1 in [500,1000,1500, 2000]:\n# for L2 in [500,1000,1500, 2000]:\n# cfg = {'Model': ['skMLP', {'Layers': [L1,L2],'learning_rate_init':learning_rate_init } ], 'Features': dict_data_cfg1 } \n# list_main_configs_models_etc += [cfg ] \n\n# 513 20*500x1000 skMLP\n# 512 10*500x1000 skMLP\n# 511 5*500x1000 skMLP\n\n# dict_data_cfg1 = {'list_features_ids': ['T5_43k', 'esm2Fairushchin_43k' , 'taxons31_onehot_43k' ] } \n# cfg = {'Model': ['skMLP', {'Layers': [500,1000] } ], 'Features': dict_data_cfg1 } \n# list_main_configs_models_etc = [cfg ]*5\n\n\n# cfg = {'Model': ['skMLP3', { } ], 'Features': dict_data_cfg10b } \n# list_main_configs_models_etc.append(cfg) \n\n# dict_data_cfg1 = {'list_features_ids': ['T5_43k', 'esm2Fairushchin_43k' , 'taxons31_onehot_43k' ] } \n# cfg = {'Model': ['LGBdefault', { } ] , 'Features': dict_data_cfg1 }\n# list_main_configs_models_etc = [ cfg ] \n\n# dict_data_cfg1 = {'list_features_ids': ['T5_43k', 'esm2Fairushchin_43k' , 'taxons31_onehot_43k' ] } \n# cfg = {'Model': ['gpuLogReg', { } ] , 'Features': dict_data_cfg1 }\n# list_main_configs_models_etc = [ cfg ] \n\n# dict_data_cfg1 = {'list_features_ids': ['T5_43k', 'esm2Fairushchin_43k' , 'taxons31_onehot_43k' ] } \n# cfg = {'Model': ['gpuCatBClasdefault', { } ] , 'Features': dict_data_cfg1 }\n# list_main_configs_models_etc = [ cfg ] \n\n# dict_data_cfg1 = {'list_features_ids': ['T5_43k', 'esm2Fairushchin_43k' , 'taxons31_onehot_43k' ] } \n# cfg = {'Model': ['gpuCatBdefault', { } ] , 'Features': dict_data_cfg1 }\n# list_main_configs_models_etc = [ cfg ] \n\n\n# dict_data_cfg1 = {'list_features_ids': ['T5_43k', 'esm2Fairushchin_43k' , 'taxons31_onehot_43k' ] } \n# cfg = {'Model': ['CatBdefault', { } ] , 'Features': dict_data_cfg1 }\n# list_main_configs_models_etc = [ cfg ] \n\n\n \n# list_main_configs_models_etc = []\n# epochs = 15\n# batch_size = 128\n\n# droupout_loc = 0.3\n# droupout_loc2 = 0.3\n# layer_size_loc1 = 2000\n\n\n# dict_data_cfg1 = {'list_features_ids': ['T5_43k', 'esm2Fairushchin_43k' , 'taxons31_onehot_43k' ] } \n# dict_data_cfg2 = {'list_features_ids': ['T5_43k', 'esm2S480_43k' , 'taxons31_onehot_43k' ] } \n# dict_data_cfg3 = {'list_features_ids': ['T5_43k', 'esm2S2560cls_43k' , 'taxons31_onehot_43k' ] } \n# dict_data_cfg4 = {'list_features_ids': [ 'T5_43k', 'E640_43k' , 'taxons31_onehot_43k'] }\n# dict_data_cfg5 = {'list_features_ids': [ 'T5_43k','E2560_43k' , 'taxons31_onehot_43k'] }\n# dict_data_cfg6 = {'list_features_ids': [ 'T5_43k','E320_43k' , 'E480_43k' , 'taxons31_onehot_43k'] }\n# dict_data_cfg7 = {'list_features_ids': [ 'T5_43k','E2560mean_43k' , 'taxons31_onehot_43k'] }\n# dict_data_cfg8 = {'list_features_ids': [ 'T5_43k','E1280_43k' , 'taxons31_onehot_43k'] }\n# dict_data_cfg9 = {'list_features_ids': [ 'T5_43k','protb_43k' , 'taxons31_onehot_43k'] }\n# dict_data_cfg10 = {'list_features_ids': [ 'E5120_43k'] }\n# dict_data_cfg10b = {'list_features_ids': [ 'E5120_43k' , 'taxons31_onehot_43k' ] }\n# dict_data_cfg11 = {'list_features_ids': ['T5_43k', 'E5120_43k', 'taxons31_onehot_43k' ] }\n\n# for LS1 in [1000]*5: # [500,500,500,500,500,500]: # list(range(100,1010,100)): \n# LS1 = int(LS1)\n# for LS2 in [2000]: # range(100,1510,300): \n# for LS3 in [None]: # range(100,1510,300): \n# for LS4 in [None]: # range(100,1510,300): \n# list_main_configs_models_etc.append( {'Model': ['KMLP', {'Layers': [LS1,LS2,LS3,LS4], 'Dropouts':[droupout_loc,droupout_loc2,droupout_loc,droupout_loc] , \n# 'BatchNormalizations':[True,False,False,False] , 'epochs' : epochs , 'batch_size' : batch_size , 'verbose' : 0 } ] } )\n# # 'Features': dict_data_cfg1 } )\n\n# # list_main_configs_models_etc = []\n# # # Best: skMLP_MI500_RS12_HL[1500, 2000]_alpha0.1 C11 0.8658 0.458203\n# cfg = {'Model': ['skMLP', {'Layers': [1500,2000] , 'alpha':0.1 } ]}# , 'Features': dict_data_cfg1 } \n# list_main_configs_models_etc += [cfg ]*10\n# # previous best - but better in blend \n# cfg = {'Model': ['skMLP', {'Layers': [500,1000] } ]}# , 'Features': dict_data_cfg1 } \n# list_main_configs_models_etc += [cfg ]*10\n \n# 527 BigMode: blend5 (basic + esm2L ) + skMLP (basic +esm2L)\n# 528 BigMode:2(basic+esm2L)+2skMLP(basic+e2L)\n# 527 BigMode:5(basic+esm2L)+5skMLP(basic+e2L)\n# 526 BigMode:5(basic+esm2L)+5skMLP(basic)\n\nprint(len(list_main_configs_models_etc)) #, list_main_configs_models_etc )","metadata":{"papermill":{"duration":0.04033,"end_time":"2023-07-15T21:44:46.029149","exception":false,"start_time":"2023-07-15T21:44:45.988819","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-20T20:34:53.982737Z","iopub.execute_input":"2023-07-20T20:34:53.98323Z","iopub.status.idle":"2023-07-20T20:34:54.004792Z","shell.execute_reply.started":"2023-07-20T20:34:53.983179Z","shell.execute_reply":"2023-07-20T20:34:54.002925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(list_main_configs_models_etc )):\n print(i)\n print(list_main_configs_models_etc[i])","metadata":{"papermill":{"duration":0.025862,"end_time":"2023-07-15T21:44:46.068129","exception":false,"start_time":"2023-07-15T21:44:46.042267","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-20T20:34:54.006866Z","iopub.execute_input":"2023-07-20T20:34:54.008614Z","iopub.status.idle":"2023-07-20T20:34:54.025861Z","shell.execute_reply.started":"2023-07-20T20:34:54.008496Z","shell.execute_reply":"2023-07-20T20:34:54.024075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Previous Outcomes\n\n\n\n### Selected outcomes\n\n See version 371 for the previous collection\n There we have:\n ## top: \n #### 201 +Taxs 5F 5F 100-2000x1000,2000 DR0.3,0.3 BN1 GPU| 3h33h\n # Previous top1 with taxons\n FinalBlend40\t0.94208\t0.492734\t0.574729\t0.386232\t0.517242\t0.356\t0.22\t0.294\t21\n\n Top for small mode ??????????\n 538 bigMode 10*(skmlp500x1000+skmlp1500x2000) basic features | 11h28m | work_mode = 'smallModeToExplore_1'work_mode = 'smallModeToExplore_1'\n blend20 0.9135 0.487265\n\n## 554 submit 0.52879 \n (!!!!!!!!!!!!) blend from 15 to 20 does not seem to improve much 0.49136 vs 0.491558 \n 554 15 smklp500x1000 bigmode | 8.5h\n FinalBlend15 0.92406 0.49136\n 553 10 smklp500x1000 bigmode\n FinalBlend10 0.92166 0.49026\n compare: \n 544 5 smklp500x1000 bigmode FinalBlend5 0.91606 0.488299 | 3h18m | \n 543 20 smklp500x1000 bigmode 11hours \n FinalBlend20 0.92508 0.491558 \n\n\n## Versions\n\n\n # 652 gpu 10* KMLP basic - standard - recheck 'epochs' : 15 ============== got almost the same number \n # FinalBlend10\n # 0.93948\n # 0.490972\n # droupout_loc = 0.3; droupout_loc2 = 0.3; \n # cfg = {'Model': ['KMLP', {'Layers': [1000,2000,None,None], 'Dropouts':[droupout_loc,droupout_loc2,droupout_loc,droupout_loc] , \n # 'BatchNormalizations':[True,False,False,False] , 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \n # list_main_configs_models_etc += [cfg ]*10\n\n # 651 gpu 10* KMLP basic with 'epochs' : 12 ------------------------------- baseline with 12 epochs \n # FinalBlend10\n # 0.94044\n # 0.490631\n\n # 650 gpu 10* dropout 0.3 0 BN1 'epochs' : 15 ---- a bit worse that previous - 12 epochs and DR0.3 ^2 \n # FinalBlend10\n # 0.93262\n # 0.486012\n\n\n # 649 gpu 10* dropout 0.3^2 BN1 'epochs' : 12 -------------- a bit better with 12 epochs \n # FinalBlend10\n # 0.93554\n # 0.487265\n\n # 648 gpu 10* dropout 0.3^2 BN1 \n # FinalBlend10\n # 0.93342\n # 0.485942\n\n # 647 with dropout 0.3^2 BN1 \n # blend11\t0.926\t0.465763\n # Best: KMLP3_L1000_L2 1.0e-07_selu_L500_L2 1.0e-07_relu_L1500_L2 1.0e-07_relu_L2000_L2 1.0e-07_relu_L2 1.0e-07 C8\t0.9154\t0.444403\n # worst: KMLP3_L1500_L2 1.0e-07_selu_L1000_L2 1.0e-07_relu_L500_L2 1.0e-07_relu_L1000_L2 1.0e-07_relu_L2 1.0e-07 C60\t0.9081\t0.418104\n\n\n # 646 with dropout 0.3 BN1 \n # Max blend58\t0.9253\t0.46444\n # Best : KMLP3_L1000_L2 1.0e-07_selu_L500_L2 1.0e-07_relu_L500_L2 1.0e-07_relu_L1500_L2 1.0e-07_relu_L2 1.0e-07 C1\t0.9142\t0.434401\n # Worst: KMLP3_L1500_L2 1.0e-07_selu_L1000_L2 1.0e-07_relu_L1500_L2 1.0e-07_relu_L1000_L2 1.0e-07_relu_L2 1.0e-07 C66\t0.9047\t0.413717\n\n\n # 644 with dropout 0.3\n # blend76\t0.93\t0.462682\n # best : KMLP3_L1500_L2 1.0e-07_selu_L1000_L2 1.0e-07_relu_L2000_L2 1.0e-07_relu_L2000_L2 1.0e-07_relu_L2 1.0e-07 C71\t0.9254\t0.455629 ---------- not bad ????\n # worst: KMLP3_L1000_L2 1.0e-07_selu_L500_L2 1.0e-07_relu_L1000_L2 1.0e-07_relu_L1500_L2 1.0e-07_relu_L2 1.0e-07 C4\t0.9219\t0.436711\n\n\n # 643 with dropout 0.5\n # blend101\t0.9299\t0.462614\n # best: KMLP3_L1000_L2 1.0e-07_selu_L1000_L2 1.0e-07_relu_L500_L2 1.0e-07_relu_L1500_L2 1.0e-07_relu_L2 1.0e-07 C13\t0.9255\t0.455482 ----------- not bad ???????????\n # worst: KMLP3_L1000_L2 1.0e-07_selu_L1000_L2 1.0e-07_relu_L2000_L2 1.0e-07_relu_L1000_L2 1.0e-07_relu_L2 1.0e-07 C21\t0.9236\t0.438\n\n\n # 642 reg1 different epochs15\n # blend39\t0.9296\t0.465333\n # best KMLP3_L2000_L2 1.0e-08_selu_L1000_L2 1.0e-06_relu_L1000_L2 1.0e-06_relu_L2000_L2 1.0e-07_relu_L2 1.0e-07 C76\t0.9249\t0.456686 ----------- not bad ???????????\n # worst KMLP3_L2000_L2 1.0e-08_selu_L1000_L2 1.0e-06_relu_L1000_L2 1.0e-06_relu_L2000_L2 1.0e-07_relu_L2 1.0e-08 C75\t0.9186\t0.439488\n\n\n # 640 LS different \n # blend88\t0.9298\t0.462205\n # best KMLP3_L1500_L2 1.0e-07_selu_L1000_L2 1.0e-07_relu_L1500_L2 1.0e-07_relu_L1000_L2 1.0e-07_relu_L2 1.0e-07 C66\t0.9252\t0.454879 ------------ not bad \n # worst: blend88\t0.9298\t0.462205\n\n\n # 639 reg1 different epochs12 \n\n\n # 638 5 * KMLP3 'epochs' : 12 --------------------------------- 12 epochs worse than 15 - compare 647 \n # blend5\n # 0.9288\n # 0.459674\n\n # 637 5 * KMLP3 recheck \n # reg1 = 1e-7; cfg = {'Model': ['KMLP3', {'Layers': [2000,1000,1000,2000],'LR':0.001,'optimizer':'adam', 'reg_l2':[reg1,reg1,reg1,reg1,reg1], 'activation':['selu','relu','relu','relu'] , \n # 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \n # blend5\n # 0.9283\n # 0.4618\n\n\n # 636 gou 'epochs' : 20 full mode [2000,1000,1000,2000],'LR':0.001 Blend10\n # FinalBlend10\n # 0.9301\n # 0.48313\n\n # 635 gpu full mode [2000,1000,1000,2000],'LR':0.001 Blend10\n # FinalBlend10 0.93548 0.486862\n\n # interactive LR':0.0006, reg1 = 1e-8; blend3\t0.9275\t0.456365\t\n\n # 634 ,'LR':0.0007 reg1 = 1e-8; 5* -------- 0.460037\n\n # 633 ,'LR':0.0007 reg1 = 1e-8; 5* -------- 0.459493\n\n # 632 ,'LR':0.0007 reg1 = 1e-8; 5* -------- 0.459126\n\n # 631 ,'LR':0.0008 reg1 = 1e-8; 5* ------ 0.460579\n\n # 630 ,'LR':0.0009 reg1 = 1e-8; 5* ------ 0.460579\n # LS = 2000; reg1 = 1e-8; cfg = {'Model': ['KMLP3', {'Layers': [2000,1000,1000,2000],'LR':0.0009,'optimizer':'adam', 'reg_l2':[reg1,reg1,reg1,reg1,reg1], 'activation':['selu','relu','relu','relu'] , \n # 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \n\n # 629 reg1 = 1e-8; swish 3relu 5* [2000,1000,1000,2000],'LR':0.001, # 0.455766 --- worse \n\n # 628 reg1 = 1e-8; swish 5* [2000,1000,1000,2000],'LR':0.001, ----- 0.455833\n\n # 627 swish 5* [2000,1000,1000,2000],'LR':0.001, --- 0.455491 - worse \n\n # 626 5*KMLP1 recheck 0.462029 ----------------------------------------------------------- baseline KMLP1 \n\n # 625 5* [2000,1000,1000,2000],'LR':0.001, # 0.462796 ----------- not worse than KMLP1 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!11\n\n\n # 624 3* [2000,1000,1000,2000],'LR':0.0005, - repeat same two times \n # blend3 0.45172\n # 0.446825 , 0.446494 , 0.44275\n\n # 623 2* [2000,1000,1000,2000],'LR':0.001, - repeat same two times \n # 0.451706\n # 0.448962\n # blend2: 0.458282\n # LS = 2000; reg1 = 1e-7; cfg = {'Model': ['KMLP3', {'Layers': [2000,1000,1000,2000],'LR':0.001,'optimizer':'adam', 'reg_l2':[reg1,reg1,reg1,reg1,reg1], 'activation':['selu','relu','relu','relu'] , \n # 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \n\n # 622[2000,1000,1000,2000],'LR':0.001, \n # 0.447809\n # LS = 2000; reg1 = 1e-7; cfg = {'Model': ['KMLP3', {'Layers': [2000,1000,1000,2000],'LR':0.001,'optimizer':'adam', 'reg_l2':[reg1,reg1,reg1,reg1,reg1], 'activation':['selu','relu','relu','relu'] , \n # 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \n # Try to reproduce: \n # 610 LS = 2000 [LS,1000,1000,LS]\n # KMLP3_L2000_L2 1.0e-07_selu_L1000_L2 1.0e-07_relu_L1000_L2 1.0e-07_relu_L2000_L2 1.0e-07_relu_L2 1.0... --------- 0.45\n # 0.9219 0.450341\n\n # 621 [5000,1000,1000,2000],'LR':0.002 \n # BAD !!! 0.430332 \n\n # 620 5 * [2000,1000,1000,2000],'LR':0.002 same blend5 times\n # 0.460829\n\n # 619 [2000,1000,1000,2000],'LR':0.002,\n # 0.435689\n # LS = 2000; reg1 = 1e-7; cfg = {'Model': ['KMLP3', {'Layers': [2000,1000,1000,2000],'LR':0.002,'optimizer':'adam', 'reg_l2':[reg1,reg1,reg1,reg1,reg1], 'activation':['selu','relu','relu','relu'] , \n # 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \n\n # 0.437124 618 ,'LR':0.005, KMLP3 [1000,1000,1000,1000]\n # 0.431367 617 ,','LR':0.005, KMLP3 [1000,1000,1000,1000]\n\n # 0.43347 616 ,'LR':0.005,' KMLP3 [1000,1000,1000,1000],\n\n # 0.443599 615 ,'LR':0.005,' KMLP3 [1000,1000,1000,1000],\n\n # 0.444361 614 ,'LR':0.002,' KMLP3 [1000,1000,1000,1000],\n\n # 0.443809 613 ,'LR':0.002,' KMLP3 [1000,1000,1000,1000],\n\n # 0.447432 612 KMLP3 [1000,1000,1000,1000],\n\n # 0.449851 611 [2000,2000,1000,1000]\n # KMLP3_L2000_L2 1.0e-07_selu_L2000_L2 1.0e-07_relu_L1000_L2 1.0e-07_relu_L1000_L2 1.0e-07_relu_L2 1.0...\n # 0.923 0.449851\n\n # 610 LS = 2000 [LS,1000,1000,LS]\n # KMLP3_L2000_L2 1.0e-07_selu_L1000_L2 1.0e-07_relu_L1000_L2 1.0e-07_relu_L2000_L2 1.0e-07_relu_L2 1.0... --------- 0.45\n # 0.9219 0.450341\n\n # 609 LS = 2000\n # KMLP3_L2000_L2 1.0e-07_selu_L2000_L2 1.0e-07_relu_L2000_L2 1.0e-07_relu_L2000_L2 1.0e-07_relu_L2 1.0...\n # 0.9229 0.445891\n\n # 608 LS = 1500 -- not good \n # KMLP3_L1500_L2 1.0e-07_selu_L1500_L2 1.0e-07_relu_L1500_L2 1.0e-07_relu_L1500_L2 1.0e-07_relu_L2 1.0...\n # 0.9207 0.440984\n # reg1 = 1e-7; cfg = {'Model': ['KMLP3', {'Layers': [1500,1500,1500,1500],'LR':0.001,'optimizer':'adam', 'reg_l2':[reg1,reg1,reg1,reg1,reg1], 'activation':['selu','relu','relu','relu'] , \n # 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \n\n # 0.458779 607 5*KMLP3\n\n # 0.465339 606 Blend 5 diffrent MLP \n\n # 604 Blend 2KMLP2+3KMLP3\n # blend5 0.9294 0.457634\n\n # 603 Blend 2KMLP2+3KMLP3\n # blend5 0.9288 0.457536\n\n # 603 lend standard params 'LR':0.001 reg1 = 1e-7; 'batch_size' : 128 , L4Concat L=1000x4 0.438718 ==== bad\n\n # 602 'LR':0.003 reg1 = 1e-7; 'batch_size' : 128 , L4Concat L=1000x4 'LR':0.003\n\n # 601 reg1 = 1e-7; 'batch_size' : 256 , L4Concat L=1000x4 'LR':0.002, -- 0.447914\n\n # 600 reg1 = 1e-7; 'batch_size' : 128 , L4Concat L=1000x4 'LR':0.002, --- 0.45059 ===================================== 0.45059 \n # {'Model': ['KMLP3', {'Layers': [1000, 1000, 1000, 1000], 'LR': 0.002, 'optimizer': 'adam', 'reg_l2': [1e-07, 1e-07, 1e-07, 1e-07, 1e-07], 'activation': ['selu', 'relu', 'relu', 'relu'], 'epochs': 15, 'batch_size': 128, 'verbose': 0}]}\n\n # 599 reg1 = 1e-7; 'epochs' : 20 L4Concat L=1000x4 e 'LR':0.001, 0.438521\n\n # 598 reg1 = 1e-6; L4Concat L=1000x4 epochs 15 'LR':0.001, 0.442574\n\n # 597 reg1 = 1e-8; L4Concat L=1000x4 epochs 15 'LR':0.001, reg1 = 1e-8; 0.441809\n\n # 596 L4Concat L=1000x4 epochs 15 'LR':0.01, 0.44602\n\n # 595 L4Concat L=1000x4 epochs 15 'LR':0.005, 0.447488 ---- good \n\n # 594 L4Concat L=1000x4 epochs 15 'LR':0.002, ---- 0.443368\n\n # 593 L4Concat L=1000x4 epochs 10 'LR':0.002, 0.44826 ----- good \n\n # 592 L4Concat L=1000x4 epochs 10 'LR':0.0002, BAD 0.403846\n\n # 591 L4Concat L=1000x4 epochs 10 'LR':0.0002,\n # 0.413429\n\n # 590 L4Concat L=1000x4 epochs 10 'LR':0.0005,\n # BAD 0.43282\n\n # 589 L4Concat L=1000x4 epochs 20 'LR':0.0005,\n # KMLP3_L1000_L2 1.0e-07_selu_L1000_L2 1.0e-07_relu_L1000_L2 1.0e-07_relu_L1000_L2 1.0e-07_relu_L2 1.0...\n # ======= 0.9247 0.446809\n\n # 588 L4Concat GPU L=1000x4 epochs 30 'LR':0.0001,\n # BAD 0.435439\n # reg1 = 1e-7; cfg = {'Model': ['KMLP3', {'Layers': [1000,1000,1000,1000],'LR':0.0001,'optimizer':'adam', 'reg_l2':[reg1,reg1,reg1,reg1,reg1], 'activation':['selu','relu','relu','relu'] , \n # 'epochs' : 30 , 'batch_size' : 128 , 'verbose' : 0 } ] } \n\n # 587 L4Concat GPU L=2000x4\n # KMLP3_L2000_L2 1.0e-07_selu_L2000_L2 1.0e-07_relu_L2000_L2 1.0e-07_relu_L2000_L2 1.0e-07_relu_L2 1.0...\n # 0.9216 0.44182\n\n # 586 L4Concat GPU L=1000x4\n # KMLP3_L1000_L2 1.0e-07_selu_L1000_L2 1.0e-07_relu_L1000_L2 1.0e-07_relu_L1000_L2 1.0e-07_relu_L2 1.0e-07 C0\t\n # 0.9248\t0.446374\n\n # 585 L4Concat GPU\n # 0.9237 0.440968 - not so good\n # reg1 = 1e-7; cfg = {'Model': ['KMLP3', {'Layers': [500,500,500,500],'LR':0.001,'optimizer':'adam', 'reg_l2':[reg1,reg1,reg1,reg1,reg1], 'activation':['selu','relu','relu','relu'] , \n # 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \n # KMLP3_L500_L2 1.0e-07_selu_L500_L2 1.0e-07_relu_L500_L2 1.0e-07_relu_L500_L2 1.0e-07_relu_L2 1.0e-07...\n\n # START 4-Layer with concact =================================================================\n\n # 583 decrese LR increase epochs reg1 = 1e-7; cfg = {'Model': ['KMLP2', {'Layers': [2000,2000],'LR':0.0001,'optimizer':'adam', 'reg_l2':[reg1,reg1,reg1,reg1,reg1], 'activation':['selu','relu'] , 'epochs' : 30 , 'batch_size' : 128 , 'verbose' : 0 } ] } \n # BAD blend5 0.9239 0.438902\n\n # 582 DR0.3 with best before -------- does not improve\n # blend5 0.9282 0.460897 \n # reg1 = 1e-7; cfg = {'Model': ['KMLP2', {'Layers': [2000,2000],'LR':0.001,'optimizer':'adam', 'reg_l2':[reg1,reg1,reg1,reg1,reg1], 'activation':['selu','relu'] , 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \n # list_main_configs_models_etc += [cfg ]*5\n\n # change LR - seesm all quite worse\n # # 581 reg1 = 1e-7; cfg = {'Model': ['KMLP2', {'Layers': [2000,2000],'LR':0.0001,'optimizer':'adam', 'reg_l2':[reg1,reg1,reg1,reg1,reg1], 'activation':['selu','relu'] , 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \n # blend4 0.9111 0.413617\n # # 580 reg1 = 1e-7; cfg = {'Model': ['KMLP2', {'Layers': [2000,2000],'LR':0.01,'optimizer':'adam', 'reg_l2':[reg1,reg1,reg1,reg1,reg1], 'activation':['selu','relu'] , 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \n # BAD\n # # 579 reg1 = 1e-7; cfg = {'Model': ['KMLP2', {'Layers': [2000,2000],'LR':0.1,'optimizer':'adam', 'reg_l2':[reg1,reg1,reg1,reg1,reg1], 'activation':['selu','relu'] , 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \n # #BAD\n # # 578 reg1 = 1e-7; cfg = {'Model': ['KMLP2', {'Layers': [2000,2000],'LR':0.1,'optimizer':'adam', 'reg_l2':[reg1,reg1,reg1,reg1,reg1], 'activation':['selu','relu'] , 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \n # change LR - seesm all quite worse\n\n\n # # 574 reg1 = 1e-7; cfg = {'Model': ['KMLP2', {'Layers': [2000,2000] , 'reg_l2':[reg1,reg1,reg1,reg1,reg1], 'activation':['selu','relu'] , 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \n # 0.460007\n # ------ reg=1e-7 seems better compare -------- : \n # # 573 reg1 = 1e-6; cfg = {'Model': ['KMLP2', {'Layers': [2000,2000] , 'reg_l2':[reg1,reg1,reg1,reg1,reg1], 'activation':['selu','relu'] , 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \n # 0.458094\n # # 565 cfg = {'Model': ['KMLP2', {'Layers': [2000,2000] , 'reg_l2':[reg1,reg1,reg1,reg1,reg1], 'activation':['selu','relu'] , 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \n # #0.45821\n\n\n # # 572 reg1 = 1e-6; cfg = {'Model': ['KMLP2', {'Layers': [2000,2000] , 'reg_l2':[reg1,reg1,reg1,reg1,reg1], 'activation':['relu','relu'] , 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \n # 0.449453\n # # 571 cfg = {'Model': ['KMLP2', {'Layers': [3000,3000] , 'reg_l2':[reg1,reg1,reg1,reg1,reg1], 'activation':['relu','relu'] , 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \n # 0.457029\n # # 570 rerun 5*skMLP1\n # 0.46604 blend14: 0.46954\n # # 569 rerun 5*KMLP \n # 0.462726\n # # 568 cfg = {'Model': ['KMLP2', {'Layers': [1000,1000] , 'reg_l2':[reg1,reg1,reg1,reg1,reg1], 'activation':['relu','relu'] , 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \n # 0.448005\n # # 567 cfg = {'Model': ['KMLP2', {'Layers': [1000,1000] , 'reg_l2':[reg1,reg1,reg1,reg1,reg1], 'activation':['selu','relu'] , 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \n # 0.456445\n # # 566 cfg = {'Model': ['KMLP2', {'Layers': [2000,2000] , 'reg_l2':[reg1,reg1,reg1,reg1,reg1], 'activation':['relu','relu'] , 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \n # 0.455287\n # # 565 cfg = {'Model': ['KMLP2', {'Layers': [2000,2000] , 'reg_l2':[reg1,reg1,reg1,reg1,reg1], 'activation':['selu','relu'] , 'epochs' : 15 , 'batch_size' : 128 , 'verbose' : 0 } ] } \n # #0.45821\n # # blend5 0.9287 0.45821\n\n 562,3 TPU trials small mode reg and LS\n \n 561 FinalBlend1 0.92528 0.428485 SAME regl2 = 1e-4 = 0.0001\n 560 FinalBlend1 0.92544 0.429491 regl2 = 1e-4 = 0.0001\n \n 559 more actiovations skip connec different regulariz small mode - small mode | 4.5H\n for a1 in ['relu','swish','selu']: # activation\n for a2 in ['relu','swish','selu']: # activation\n for reg1 in [1e-8,1e-6, 1e-4,1e-2]:\n \n KMLP2 C25 0.9247 0.451642\n ['KMLP2', {'Layers': [2000, 2000], 'reg_l2': [1e-06, 1e-06, 1e-06, 1e-06, 1e-06], 'activation': ['selu', 'relu'], 'epochs': 15, 'batch_size': 128, 'verbose': 0}] \n KMLP2 C12 0.9267 0.451396\n ['KMLP2', {'Layers': [2000, 2000], 'reg_l2': [1e-08, 1e-08, 1e-08, 1e-08, 1e-08], 'activation': ['swish', 'relu'], 'epochs': 15, 'batch_size': 128, 'verbose': 0}] \n KMLP2 C24 0.9243 0.451339\n ['KMLP2', {'Layers': [2000, 2000], 'reg_l2': [1e-08, 1e-08, 1e-08, 1e-08, 1e-08], 'activation': ['selu', 'relu'], 'epochs': 15, 'batch_size': 128, 'verbose': 0}]\n KMLP2 C5 0.9263 0.449429\n ['KMLP2', {'Layers': [2000, 2000], 'reg_l2': [1e-06, 1e-06, 1e-06, 1e-06, 1e-06], 'activation': ['relu', 'swish'], 'epochs': 15, 'batch_size': 128, 'verbose': 0}]\n\n TAIL:\n KMLP2 C19 0.8452 0.307635\n {'Model': ['KMLP2', {'Layers': [2000, 2000], 'reg_l2': [0.01, 0.01, 0.01, 0.01, 0.01], 'activation': ['swish', 'swish'], 'epochs': 15, 'batch_size': 128, 'verbose': 0}]}\n \n 558 more actiovations skip connec different regulariz small mode - small mode 2H 4MIN \n for a1 in ['relu','swish','selu']: # activation\n for a2 in ['relu','swish','selu']: # activation\n for reg1 in [1e-8, 1e-4]: \n \n KMLP2 C12 0.9264 0.453821\n {'Model': ['KMLP2', {'Layers': [2000, 2000], 'reg_l2': [1e-08, 1e-08, 1e-08, 1e-08, 1e-08], 'activation': ['selu', 'relu'], 'epochs': 15, 'batch_size': 128, 'verbose': 0}]} \n\n blend17 0.9281 0.450803\n KMLP2 C8 0.926 0.450767\n ['KMLP2', {'Layers': [2000, 2000], 'reg_l2': [1e-08, 1e-08, 1e-08, 1e-08, 1e-08], 'activation': ['swish', 'swish'], 'epochs': 15, 'batch_size': 128, 'verbose': 0}] \n\n KMLP2 C7 0.9028 0.385415\n ['KMLP2', {'Layers': [2000, 2000], 'reg_l2': [0.0001, 0.0001, 0.0001, 0.0001, 0.0001], 'activation': ['swish', 'relu'], 'epochs': 15, 'batch_size': 128, 'verbose': 0}] \n KMLP2 C9 0.8978 0.372249 \n ['KMLP2', {'Layers': [2000, 2000], 'reg_l2': [0.0001, 0.0001, 0.0001, 0.0001, 0.0001], 'activation': ['swish', 'swish'], 'epochs': 15, 'batch_size': 128, 'verbose': 0}] \n KMLP2 C3 0.9055 0.382113\n ['KMLP2', {'Layers': [2000, 2000], 'reg_l2': [0.0001, 0.0001, 0.0001, 0.0001, 0.0001], 'activation': ['relu', 'swish'], 'epochs': 15, 'batch_size': 128, 'verbose': 0}] \n \n 557 more actiovations skip connec different regulariz small mode - big mode ------ CRASH RAM \n \n 556 actiovations skip connec different regulariz small mode- big mode --- 8-hours\n FinalBlend8 0.93874 0.482059\n \n 555 skip connec different regulariz small mode- big mode - RAM CRASH \n \n## 554 submit 0.52879 \n 554 15 smklp500x1000 bigmode | 8.5h\n FinalBlend15 0.92406 0.49136\n 553 10 smklp500x1000 bigmode\n FinalBlend10 0.92166 0.49026\n compare: \n 544 5 smklp500x1000 bigmode FinalBlend5 0.91606 0.488299 | 3h18m |\n 543 20 smklp500x1000 bigmode \n\n 552 5 KMLP rerun small mode\n blend5 0.9267 0.462508 mean 0.9185 0.4445\n \n 551 rerun skMLP1*5\n blend5 0.8689 0.46457 mean 0.83912 0.4488 \n\n 550 KMLP2 with skip connections 2000,2000 size Quick Save \n blend5\t0.9286\t0.454546\t\n mean\t0.925780\t0.447121\t\n \n 549 KMLP2 with skip connections 2000,2000 size\n blend5 0.9282 0.454491 \n \n 548 KMLP2 with skip connections first run 1000,1000 size\n blend5 0.9275 0.449062\n \n \n 547 rerun 5 skmlp basic ------ incorrect params not 10_000 but full samples \n blend5 0.9042 0.483855\n 546 rerun 5 kmlp basic ------ incorrect params not 10_000 but full samples \n blend5 0.9331 0.481272\n \n 544 5 smklp500x1000 bigmode FinalBlend5 0.91606 0.488299 | 3h18m | \n \n 543 20 smklp500x1000 bigmode 11hours \n FinalBlend20 0.92508 0.491558 \n\n 542 bigMode 10*(skmlp500x1000+skmlp1500x2000 + KMLP) basic features | RAM crash 1 hour \n \n 541 small mode 10*(skmlp500x1000+skmlp1500x2000 + KMLP) basic features / 12h timelimit \n \n 538 bigMode 10*(skmlp500x1000+skmlp1500x2000) basic features | 11h28m | work_mode = 'smallModeToExplore_1'work_mode = 'smallModeToExplore_1'\n blend20 0.9135 0.487265\n\n# =============== top for small mode !!!!!!!!!!!!!!!!!\n\n 537 20*skmlp500x1000 basic features blend20 0.8742 0.468902 ---------- top result better than other skmlp and keras and blends of \n Compare to KMLP:\n 429 rechecks 20*basic config 2h5m Mean 0.448304 Min 0.439531 Max 0.448304\n blend20 0.9282 0.466223\n 428 rechecks 20*basic config 2h22m Mean 0.44533 Min 0.438931 Max 0.451435\n blend20 0.928 0.46739\n \n \n 536 10 skmlp1 + skmlp2\n blend20 0.8828 0.467865 --------------------------------------quite top result \n Compare: 534 - just skmlp2 - blend20 0.8869 0.464403\n And:\n 429 rechecks 20*basic config 2h5m Mean 0.448304 Min 0.439531 Max 0.448304\n blend20 0.9282 0.466223\n 428 rechecks 20*basic config 2h22m Mean 0.44533 Min 0.438931 Max 0.451435\n blend20 0.928 0.46739\n\n 534 run best Small Mdde, skMLP * 20\n blend20 0.8869 0.464403\n \n 533 run best Small Mdde, skMLP * 10 | 1h19m | a bit grows over 10 ----- worse than previos top skmlp 500x1000\n blend10 0.8857 0.463748\n Compare\n 517 blend10 0.8719 0.466181 skMLP1 10* 500x1000 small mode basic feat\n\n\n \n 532 run best Small Mdde, skMLP * 5 \n blend5 0.8872 0.462618 -------------- seems WORSE than previos slMLP - 5160 ------------- and only bit worse KMLP \n From: 489 skMLP and also alpha 1e-1:1e-4 FinalBlend80 0.8932 0.466266 | 10h27\n # Best: skMLP_MI500_RS12_HL[1500, 2000]_alpha0.1 C11 0.8658 0.458203\n cfg = {'Model': ['skMLP', {'Layers': [1500,2000] , 'alpha':0.1 } ], 'Features': dict_data_cfg1 } \n list_main_configs_models_etc += [cfg ]*5\n Comapre:\n Compare:\n 516 skMLP1 5* 500x1000 small mode basic feat\n blend5 0.8622 0.465465 ----------------------------- better than KMLP in F1\n Compare:\n 425 rechecks 5*basic config 36Min| Mean: 0.44547 Max 0.446886 Min 0.4445\n blend5 0.9261 0.463558\n 424 rechecks 5*basic config 38Min| Mean: 0.4448 Max 0.448598 Min 0.439295\n blend5 0.9262 0.462999\n \n\n \n\n 528 BigMode:2(basic+esm2L)+2skMLP(basic+e2L) | crash 2 hours \n 527 BigMode: blend5 (basic + esm2L ) + skMLP (basic +esm2L) | 12 h timelimit \n 526 BigMode: blend5 (basic + esm2L ) + 5 skMLP (basic) FinalBlend15 0.93688 0.49687 |7h38\n \n 525 BigMode: blend5 (basic + esm2L ) | CRASH RAM no output\n 524 BigMode: blend5 (basic + esm2L ) CRASH RAM no output\n repeat of (with scoring :\n 494 BigMode: blend5 (basic + esm2L ) | 7h\n 493 BigMode: blend4 (basic + esm2L ) | 5h21min \n 492 BigMode: blend3 (basic + esm2L ) RAM CRASH \n\n 523 5 *( skMLP ) basic features - full mode \n FinalBlend5 0.91706 0.488856 | 3.5h \n \n 522 5 (skMLP ) basic features - full mode \n FinalBlend5 0.91568 0.488613 \n\n \n 521 5 *skMLP + 5*KMLP basic features - full mode \n See small mode --- 519 ---- blend10 0.8986 0.470254 ------------------------------------ 0.47 - quite good - let us try to sumbit \n\n 520 5 skMLP ['T5_43k', 'E5120_43k', 'taxons31_onehot_43k' \n blend5 0.8553 0.456003 -------- worse \n\n 519 5 *skMLP + 5*KMLP basic features \n blend10 0.8986 0.470254 ------------------------------------ 0.47 - quite good - let us try to sumbit \n\n 518 skMLP1 20* 500x1000 small mode basic feat ------------ from 10 to 20 is small uplift\n blend20 0.8727 0.468125\n \n 517 blend10 0.8719 0.466181 skMLP1 10* 500x1000 small mode basic feat\n \n 516 skMLP1 5* 500x1000 small mode basic feat\n blend5 0.8622 0.465465 ----------------------------- better than KMLP in F1\n Compare:\n 425 rechecks 5*basic config 36Min| Mean: 0.44547 Max 0.446886 Min 0.4445\n blend5 0.9261 0.463558\n 424 rechecks 5*basic config 38Min| Mean: 0.4448 Max 0.448598 Min 0.439295\n blend5 0.9262 0.462999\n\n 511` skMLP1 5* 500x1000 small mode basic feat: Mean: 0.449668 Min: 0.447936 Max 0.452763\n (!) mean is better for 500x1000 compare to 1000x2000 - 509: mean: 0.4418\n Compare to basic config:\n 431 rechecks 50*basic config 6h37m Mean 0.445 Min 0.4389 Max 0.4495 Std 0.0026\n blend37 0.9282 0.467836 \n \n\n 510 skMLP1 20* 1000x2000 small mode basic feat mean 0.81784 0.44086155 \n 509 skMLP1 10* 1000x2000 small mode basic feat mean: 0.4418\n 508 skMLP1 5* 1000x2000 small mode basic feat mean: 0.4424308\n\n 505 gpuLR on ordered targets - 500 - again crash \n\n 504 catboost on 10 targets - fail train targets are equal \n \n 503 gpuLR on ordered targets - 1850 - same crash \n \n 502 gpuCatBClasdefault Y50 # gpuCatBClasdefault C0 0.518 0.005369 | 1h23min\n\n 499 gpuLogReg small config1850 --- crash - may be problem in target with only zeros \n \n 497 gpuLogReg C0 0.6378 0.064386 gpuLogReg Y100 | 2min\n \n 496 LGBdefault C0 0.8409 0.192439 Small mode: test LGB Y100 | 46 min\n \n \n 494 BigMode: blend5 (basic + esm2L ) | 7h\n 493 BigMode: blend4 (basic + esm2L ) | 5h21min \n 492 BigMode: blend3 (basic + esm2L ) RAM CRASH \n 491 BigMode: blend2 (basic + esm2L ) RAM CRASH \n\n 490 change also LR skMLP blend57 0.8725 0.467746 | 10h27\n Best skMLP_MI500_RS45_HL[500, 1000]_alpha0.0001_LR0.001 C33 0.8418 0.45281\n Worst skMLP_MI500_RS88_HL[1500, 2000]_alpha0.0001_LR0.001 C75 0.8167 0.438259\n \n 489 skMLP and also alpha 1e-1:1e-4 FinalBlend80 0.8932 0.466266 | 10h27\n Best: skMLP_MI500_RS12_HL[1500, 2000]_alpha0.1 C11 0.8658 0.458203\n Worst skMLP_MI500_RS36_HL[2000, 2000]_alpha1e-05 C79 0.8051 0.432196\n \n 488 skMLP [500,1000,1500,2000]^2 small mode basic features 2h 27 min \n blend15 0.8693 0.467042 \n Best: kMLP_MI500_RS63_HL[500, 1000]_alpha0.0001 C1 0.8361 0.448364\n Worst: skMLP_MI500_RS80_HL[1500, 2000]_alpha0.0001 C11 0.81 0.434975\n \n FinalBlend16 0.8691 0.466792 --------------------------------------------------- result in F1 compatible with KMLP !!!!!!!!!!!!!!!!\n Compare:\n 429 rechecks 20*basic config 2h5m Mean 0.448304 Min 0.439531 Max 0.448304\n blend20 0.9282 0.466223\n\n 486 crash skMLP1000-2000 small mode basic features\n \n 485 FinalBlend5 0.8825 0.436434 skML3 small mode basic features\n 484 FinalBlend5 0.8972 0.438343 repeat skML2 small mode basic features\n 483 FinalBlend5 0.8983 0.435959 skML2 small mode basic features\n\n============== Test different models ============== \n\n 482 BigMode: blend5 (basic + esm2L ) effectively 20 blends | gpu RAM CRASH\n 481 BigMode: blend10 (basic + esm2L ) effectively 20 blends | gpu RAM CRASH\n \n 480 blend19 0.9295 0.468744 blend10 (basic + esm2L ) effectively 20 blends ----- very small gain to basic\n Compare:\n 429 rechecks 20*basic config 2h5m Mean 0.448304 Min 0.439531 Max 0.448304\n blend20 0.9282 0.466223\n 428 rechecks 20*basic config 2h22m Mean 0.44533 Min 0.438931 Max 0.451435\n blend20 0.928 0.46739\n \n \n 479 blend10 0.9287 0.466801 blend9 0.9287 0.467149 blend5 basic + esm2L effectively 10 blends - not better than basic blend 10 !!! (((((((((((\n Compare:\n 427 rechecks 10*basic config 1h9m| Mean 0.446455 Max 0.4492 Min 0.443717\n blend10 0.9269 0.466354\n 426 rechecks 10*basic config 1h9m| Mean: 0.44436 Max 0.446033 Min 0.441857\n blend10 0.927 0.46432\n \n \n \n 478 try sklearn MLP skMLP_MI500_RS1 C0 0.889 0.402608\n \n 477 blend5 0.9179 0.446579 no t5: esm2L+tax basic MLP\n \n 475 blend5 0.9264 0.464552 rerun basic t5+...\n \n \n 474 blend5 0.9236 0.455406 relu-selu-sigmoid basic-baseline T5+Fairushin+Tax KMLP basic ---- SELU - worse \n 473 blend5 0.9232 0.461322 selu-relu-sigmoid basic-baseline T5+Fairushin+Tax KMLP basic\n 472 blend5 0.9236 0.461574 sigmoid-relu-sigmoid basic-baseline T5+Fairushin+Tax KMLP basic GPU\n 471 blend5 0.9263 0.452557 relu-sigmoid-sigmoid basic-baseline T5+Fairushin+Tax KMLP basic\n 470 blend5 0.9261 0.462416 repeat basic-baseline T5+Fairushin+Tax KMLP basic\n \n \n 469 blend5 0.92 0.453464 repeat baseline 5*esm2Large+basic KMLP basic\n 468 blend5 0.9153 0.442981 sigmoid all 5*esm2Large+basic KMLP basic\n 467 blend5 0.9197 0.453946 repeat baseline 5*esm2Large+basic KMLP basic\n 466 blend5 0.9234 0.452228 10 epochs 5*esm2Large+basic KMLP basic -- 10 epochs a bit worse 20 \n 465 blend5 0.6243 0.273305 reluAll 5*esm2Large+basic KMLP basic ------ terrible reluALL\n 464 blend5 0.9222 0.443504 5*esm2Large+basic KMLP L3 2000^4 ------------------------------- four layers - not so bad, but worse L2 \n 463 blend5 0.9222 0.443504 5*esm2Large+basic KMLP L3 2000^3------------------------------- 3 layers - not so bad, but worse L2\n 462 blend5 0.923 0.451899 5*esm2Large+basic KMLP-basic 1000x1000\n 461 blend3 0.8661 0.346957 5*esm2Large+basic KMLP-basic batch1024 --- terrible batchsize1024\n 460 blend5 0.9151 0.447927 5*esm2Large+basic KMLP-basic 1000x2000 DR0.1 - less droupout worse \n 459 blend5 0.9059 0.446293 5*esm2Large+basic KMLP-basic 1000x2000 DR0.3 epoch 30 - 30 epochs worse \n 458 blend5 0.9143 0.452397 5*esm2Large+basic KMLP-basic 1000x2000 DR0.3 epoch 20 --- 20 epochs worse \n 457 blend5 0.9211 0.451444 5*esm2Large+basic KMLP-basic 500x2000 DR0.3 - first layer decrease - worse \n 456 blend5 0.9138 0.451991 5*esm2Large+basic KMLP-basic 3000x3000 DR0.5 - bigger dropout better for bigger sizes \n 455 blend5 0.9119 0.449956 5*esm2Large+basic KMLP-basic 3000x3000 - large sizes - worse \n Baseline basic config:\n 446 blend5 0.9212 0.454751 5*esm2Large+basic KMLP-basic small mode\n 467 blend5 0.9197 0.453946 repeat baseline 5*esm2Large+basic KMLP basic\n 469 blend5 0.92 0.453464 repeat baseline 5*esm2Large+basic KMLP basic\n Worse baseline \n 477 blend5 0.9179 0.446579 no t5: esm2L+tax basic MLP\n\n\n 454 blend5 0.9155 0.442438 5*esm2Large small mode L3 2000^3 DR0.3|GPU\n 453 blend5 0.9168 0.444959 5*esm2Large small mode L2 1000x2000 DR0.6 ---------- increase dropout - worse\n 452 blend5 0.915 0.448075 5*esm2Large small mode L2 1000x2000 DR0.5 ---------- increase dropout - worse\n 451 blend5 0.9077 0.443311 5*esm2Large small mode L2 3000x4000 DR0.5 - increase layers + dropout -worse \n 450 blend5 0.9029 0.435731 5*esm2Large small mode L2 3000x4000 --------- hence higher layers - worse \n 449 blend5 0.9048 0.438673 5*esm2Large small mode L2 3000x3000 ---------- hence higher layers - worse \n 448 blend5 0.9099 0.442652 5*esm2Large small mode L2 1000x3000 | mean 0.421295 ---------- L2 = 3000 seems a bit worse compare to 447\n 447 blend5 0.9107 0.442098 5*esm2Large KMLP-basic small mode | mean 0.90086 0.423505\n 446 blend5 0.9212 0.454751 5*esm2Large+basic KMLP-basic small mode\n 445 Ridge100 0.9074 0.442253 5*esm2Large+basic Ridge100 small mode 30 min | results exactly the same - blend changes nothing\n 444 Ridge100 0.9074 0.442253 4*esm2Large+basic Ridge100 small mode| results exactly the same - blend changes nothing\n 443 Ridge100 0.8978 0.435118 esm2Large Ridge100 small mode | 5min\n\n=== esm2large - tests in small mode\n\n 440 LB 0.4915 !!!!!!! Ridge100 esm2Large 5folds with submit mode\n\n# ESM2Large - good result on LB ============== esm2Large !!!!!!!!! wow we get result seems better than just t5 \n\n=======Below - return to Y1850-Zoltan; recheck small-mode =========\n\n (!) Only blend50 seems to be stable up to 4 digits , blend5-10-20 unstable about 0.001-0.002\n\n 431 rechecks 50*basic config 6h37m Mean 0.445 Min 0.4389 Max 0.4495 Std 0.0026\n blend37 0.9282 0.467836 \n blend50 0.9284 0.467601\n 430 rechecks 50*basic config 5h22m Mean 0.4457 Min 0.441527 Max 0.45\n blend50 0.9287 0.467653 \n \n 429 rechecks 20*basic config 2h5m Mean 0.448304 Min 0.439531 Max 0.448304\n blend20 0.9282 0.466223\n 428 rechecks 20*basic config 2h22m Mean 0.44533 Min 0.438931 Max 0.451435\n blend20 0.928 0.46739\n \n 427 rechecks 10*basic config 1h9m| Mean 0.446455 Max 0.4492 Min 0.443717\n blend10 0.9269 0.466354\n 426 rechecks 10*basic config 1h9m| Mean: 0.44436 Max 0.446033 Min 0.441857\n blend10 0.927 0.46432\n \n 425 rechecks 5*basic config 36Min| Mean: 0.44547 Max 0.446886 Min 0.4445\n blend5 0.9261 0.463558\n 424 rechecks 5*basic config 38Min| Mean: 0.4448 Max 0.448598 Min 0.439295\n blend5 0.9262 0.462999\n \n# Recheck basic congigs in small mode \n \n======= return to Y1850-Zoltan; recheck small-mode =========\n\n 423 FinalBlend1 0.927 0.442501 rerun Y3000\n 422 FinalBlend1\t0.9245\t0.435411 Y3000 \n 421 FinalBlend1\t0.9438\t0.442359\t Y8000\n 420 FinalBlend1\t0.9328\t0.446182 Y4000ordered small-mode \n 419 FinalBlend1\t0.9139\t0.44346 Y1850ordered small-mode \n 418 FinalBlend1\t0.9189\t0.448437\t Y1850Zoltan \n 417QS FinalBlend1\t0.9238\t0.444641\t Y2850 small-mode Quick Save \n 416 FinalBlend1\t0.9259\t0.448728\t Y2850 small-mode \n \n# It seems result are negative - score does not improve with more tartgets also Zoltan's selection seems to be better than straighforward \n# Increase targets number test in small mode first (i.e. 1 fold out of 2, and 10_000 samples) \n \n 413 FinalBlend1 0.9303 0.470933 1*cfg1 basic recheck | GPU | threshold 0.1 \n 412 1*сfg4 'E640_43k' \n 411 LB 0.33102 FinalBlend1 0.9273 0.464511 1*cfg9 (protb)| gpu 27min\n 410 FinalBlend1 0.92736 0.461956 1*cfg9 (protb) | cpu | 1h13min\n 409 1*cfg8 (esm21280-Andrey)|gpu| no scoring|\n 408 LB 0.2938 5*cfg8 (esm21280-Andrey)|gpu| no scoring| 0.18 threshold \n 407 FinalBlend10 0.93968 0.49078 10*cfg8 (esm21280-Andrey)|gpu | ram crash on submission part (2h)\n 406 crash no results saved 1 hour\n 405 LB 0.29359 FinalBlend10 0.9342 0.470611 10*cfg7 (e2s2560mean ) full run for submit - 5 folds, 43189 samples | gpu | 1h19min\n 404 10*cfg2 (e2s2560 ) full run for submit - 5 folds, 43189 samples | gpu | \n \n ====== above basic model is used - L2 1000x2000 DR0.3^2, BN1 and only middle features are changing \n \n# Submits with esm2560 ==================== for sumbit ============\n\n 403 FinalBlend20 0.9279 0.466535 20*basic config \n 402 FinalBlend30 0.9283 0.466795 30*basic config \n \n 401 FinalBlend20 0.9268 0.464667 20*cfg7 E2560mean_43k cpu\n 400 FinalBlend20 0.9267 0.465891 20*cfg7 E2560mean_43k gpu ---- similar to basic \n -------- new config7 - should be same as cfg5\n \n 399 FinalBlend30 0.9313 0.47221 10*cfg1-3 usual DR gpu - compare to 380-383\n 398 FinalBlend30 0.9311 0.473143 10*cfg1-3 usual DR gpu - compare to 380-383\n Dropout seems to be important after the first layer - but results similar to both DR 0.3 \n 398 DR 0.3 0.3 FinalBlend30 0.9311 0.473143 GPU\n 399 DR 0.3 0.3 FinalBlend30\t0.9313\t0.47221 GPU\n \n 396-384with gpu FinalBlend10 0.9259 0.464206 e480\n\n#### From gpu-20* and cpu 10* conclusions more or less similar cfg2=e480,cfg4=e640 - top2,3 places \n \n 395 top4 FinalBlend20 0.9264 0.462629 20cfg6 e320 e480 gpu 1F2 s10k\n 394 top5 FinalBlend20 0.9261 0.464243 20cfg5 E2560 gpu 1F2 s10k\n 393 top3 FinalBlend20 0.9271 0.464794 20cfg4 E640\n 392 tail FinalBlend20 0.9241 0.457481 20cfg3 esm2S2560cls\n 391 top2 FinalBlend20 0.9269 0.465074 20cfg2 esm2S480\n 390 top1 FinalBlend20 0.9283 0.467096 20cfg1\n \n 389 top1 FinalBlend10 0.9275 0.466938 cfg1 gpu\n 388 top5 FinalBlend10 0.9257 0.460943 cfg6 10e320e480 cpu\n 387 top4 FinalBlend10 0.926 0.463132 cfg5 10e2560 cpu\n 386 top2 FinalBlend10 0.9265 0.46453 cfg4 10e640 cpu\n 385 tail FinalBlend10 0.9226 0.45513 cfg3 10e2560cls cpu \n 384 top3 FinalBlend10 0.9259 0.464071 cfg2 e480\n 396=384with gpu FinalBlend10 0.9259 0.464206 e480\n\n#### Check configs one by one\n\n#### Dropout seems to be important after the first layer\n\n 383 DR 0 0.5 FinalBlend30 0.9307 0.470698 \n 382 DR 0.5 0 FinalBlend30 0.9312 0.472475 \n 381 DR 0 0.3 FinalBlend30 0.929 0.46913 \n 380 DR 0.3 0 FinalBlend30 0.9298 0.473065\n Dropout seems to be important after the first layer - but results similar to both DR 0.3 \n 398 DR 0.3 0.3 FinalBlend30 0.9311 0.473143 GPU\n 399 DR 0.3 0.3 FinalBlend30\t0.9313\t0.47221 GPU\n \n===== Dropout seems to be important after the first layer\n\n ----- all above on 1-fold-2, samples 10_000 \n \n 377 check new basic config 20*1000x2000 5F|GPU| FinalBlend20 0.94014 0.49202 LB 0.54738\n 376 Full run 10* 6Cfgs | RAM CRASH 1h \n 375 Full run 1* 6Cfgs| 1h | FinalBlend6\t0.94066\t0.494007 | LB 0.35468\n Compare:\n #### Previous Top1: 201 +Taxs 5F 5F 100-2000x1000,2000 DR0.3,0.3 BN1 GPU| 3h33h\n # Previous top1 with taxons\n FinalBlend40 0.94208 0.492734 0.574729 0.386232 0.517242 0.356 0.22 0.294 21\n\n## Sumbits \n \n 374 FinalBlend6\t0.9295\t0.47081 Compare to 371 - more or less similar \n\n#### 372 Cleaned everything added load X_submit \n\n#### 371 1*6cfgs (1F2,S10_000) \n FinalBlend6 0.9301 0.471176\n \n 361: FinalBlend60 0.9319 0.473229 basic 2L 10*6CFG_b include both 'E2560_43k' and E+E (small) \n 357 basic 2L 10*FiveCFG GPU | 10m FinalBlend50 0.9313 0.474179\n 356 basic 2L 5*FiveCFG GPU | 7m FinalBlend25 0.931 0.47211","metadata":{}},{"cell_type":"markdown","source":"# Preparations and data load","metadata":{"papermill":{"duration":0.013206,"end_time":"2023-07-15T21:44:46.096049","exception":false,"start_time":"2023-07-15T21:44:46.082843","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport time\nt0start = time.time()\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n\ntry : \n import matplotlib.pyplot as plt\n import seaborn as sns # there is no one for TPU machine \nexcept: \n print('Expeption importing matplotlib or seaborn')\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":1.103043,"end_time":"2023-07-15T21:44:47.212744","exception":false,"start_time":"2023-07-15T21:44:46.109701","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-20T20:34:54.02852Z","iopub.execute_input":"2023-07-20T20:34:54.029359Z","iopub.status.idle":"2023-07-20T20:34:54.093225Z","shell.execute_reply.started":"2023-07-20T20:34:54.029034Z","shell.execute_reply":"2023-07-20T20:34:54.091279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Load ","metadata":{"papermill":{"duration":0.014846,"end_time":"2023-07-15T21:44:47.243589","exception":false,"start_time":"2023-07-15T21:44:47.228743","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\n\n\nif '43k' in str(list_features):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/df_folds_43k_Nsims6_evalue0d001.csv'\n df_folds = pd.read_csv(fn,index_col = 0)\n df_folds = df_folds.iloc[:n_samples_to_consider,:]\n n_samples_to_consider = len(df_folds) # In case n_samples_to_consider is greater than actual sample number we cut it down \n print('df_folds', df_folds.shape )\n display(df_folds.head(3) )\n \n print()\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_ids_cut43k.npy'\n vec_train_protein_ids = np.load(fn)\n print('vec_train_protein_ids:', vec_train_protein_ids.shape)\n print( vec_train_protein_ids[:10] )\n vec_train_protein_ids = vec_train_protein_ids[:n_samples_to_consider]\n print('Check indexes coincide, expect zero: ', (vec_train_protein_ids != df_folds.index.values).sum() )\n\n \n if mode_Y_source_files_to_use == 'Y1850':\n print()\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/Y_1850_Zoltan_cut43k.npy'\n Y = np.load(fn).astype(np.float16)\n print('Y', Y.shape, 'loaded')\n Y = Y[:n_samples_to_consider,:n_labels_to_consider]\n n_labels_to_consider = Y.shape[1] # in case n_labels_to_consider is greater that Y.shape we decrease it\n print('Y', Y.shape, 'effective')\n print(Y[:3,:2] ) \n\n print()\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/Y_labels_1850_Zoltan.csv'\n Y_labels = pd.read_csv(fn).iloc[:n_labels_to_consider,:]\n print('Y_labels', Y_labels.shape)\n display(Y_labels.head(3) )\n\n print()\n labels_to_consider = Y_labels.iloc[:,0].values\n print('labels_to_consider', len(labels_to_consider), labels_to_consider[:15]) \n \n elif mode_Y_source_files_to_use == 'Y31466':\n from scipy import sparse\n fn = '/kaggle/input/protein-embeddings-4-cafa5-selected/Y31466_sparse_float32_cut43k.npz'\n Y = sparse.load_npz(fn )\n print('Y', Y.shape, 'loaded')\n Y = Y[:n_samples_to_consider,:n_labels_to_consider].toarray()\n n_labels_to_consider = Y.shape[1] # in case n_labels_to_consider is greater that Y.shape we decrease it\n print('Y', Y.shape, 'effective')\n print(Y[:3,:2] ) \n \n print()\n fn = '/kaggle/input/protein-embeddings-4-cafa5-selected/Y31466_terms_ids.npy'\n Y_labels = np.load(fn, allow_pickle=True )[:n_labels_to_consider]\n print('Y_labels', Y_labels.shape, 'effective shape')\n print(Y_labels[:10] )\n \n print()\n labels_to_consider = Y_labels\n print('labels_to_consider', len(labels_to_consider), labels_to_consider[:15]) \n \n else :\n print('Unknown mode_Y_source_files_to_use', mode_Y_source_files_to_use)\n raise Exception(\"Unknown mode_Y_source_files_to_use\")\n \n\n \nelse:\n \n ######################################################################################3\n ############### Load trainIds\n ######################################################################################3\n\n fn = '/kaggle/input/t5embeds/train_ids.npy'\n print(fn)\n vec_train_protein_ids = np.load(fn)\n print(vec_train_protein_ids.shape)\n print( vec_train_protein_ids[:10] )\n \n\nif list_features == ['T5_43k']:\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n X = np.load(fn)\n print('X.shape', X.shape, 'loaded')\nelif set(list_features) == set(['T5_43k', 'esm2Fairushchin_43k']):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_esm2Fairushchin_float32_orderedAsInput_cut43k.npy'\n X = np.concatenate( [np.load(fn), np.load(fn2)], axis =1)\nelif set(list_features) == set(['T5_43k', 'esm2Fairushchin_43k','taxons31_onehot_43k']):\n from sklearn.decomposition import TruncatedSVD\n from sklearn.preprocessing import MinMaxScaler\n scaler = MinMaxScaler()\n reducer = TruncatedSVD(n_components=50, n_iter=7, random_state=42)\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_esm2Fairushchin_float32_orderedAsInput_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n# X = np.concatenate( [np.load(fn),scaler.fit_transform(reducer.fit_transform( np.load(fn) )).astype(np.float32), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32)], axis =1)\n# X = scaler.fit_transform(X).astype(np.float32)\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32)], axis =1)\n# X = scaler.fit_transform(X).astype(np.float32)\n \nelif set(list_features) == set(['T5_43k', 'esm2Fairushchin_43k','taxons31_onehot_43k', 'F06_genes' ]):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_esm2Fairushchin_float32_orderedAsInput_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n fn4 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features06_genes_names_related_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),\n pd.read_csv(fn4,index_col = 0).values.astype(np.float32) ], axis =1)\nelif set(list_features) == set(['T5_43k', 'esm2Fairushchin_43k','taxons31_onehot_43k', 'F07_DesriptKw1H' ]):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_esm2Fairushchin_float32_orderedAsInput_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n fn4 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features07_keywords_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),\n pd.read_csv(fn4,index_col = 0).values.astype(np.float32) ], axis =1)\n\nelif set(list_features) == set(['T5_43k', 'esm2S2560_43k', 'taxons31_onehot_43k' ]):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t36B3S2560_train_embeds_float32_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),\n ], axis =1)\nelif set(list_features) == set(['T5_43k', 'esm2S1280_43k', 'taxons31_onehot_43k' ]):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t33M650S1280_train_embeds_float32_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),\n ], axis =1)\n\nelif set(list_features) == set(['T5_43k', 'esm2S480_43k', 'taxons31_onehot_43k' ]):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t12M35S480_train_embeds_float32_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),\n ], axis =1)\nelif set(list_features) == set(['T5_43k', 'protb_43k', 'taxons31_onehot_43k' ]):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_protb_train_float32_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),\n ], axis =1)\nelif set(list_features) == set([ 'T5_43k', 'esm2S480_43k' , 'protb_43k' , 'taxons31_onehot_43k']):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t12M35S480_train_embeds_float32_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n fn4 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_protb_train_float32_cut43k.npy'\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),\n np.load(fn4), ], axis =1)\nelif set(list_features) == set([ 'T5_43k', 'esm2S2560cls_43k' , 'taxons31_onehot_43k']):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t36B3S2560cls_train_float32_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),], axis =1)\n #dict_data_cfg3 = {'list_features_ids': ['T5_43k', 'esm2S2560cls_43k' , 'taxons31_onehot_43k' ] } \n # list_features = ['T5_43k', 'esm2S2560cls_43k' , 'protb_43k' , 'taxons31_onehot_43k' ] # , 'F06_genes' # ['T5_43k']# \n # /kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t36B3S2560cls_train_float32_cut43k.npy\n\nelif set(list_features) == set([ 'T5_43k', 'E640_43k' , 'taxons31_onehot_43k']):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t30M150S640_train_embeds_float32_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),], axis =1)\n# list_features = ['T5_43k', 'E640_43k' , 'taxons31_onehot_43k' ] # , 'F06_genes' # ['T5_43k']# \n # /kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t30M150S640_train_embeds_float32_cut43k.npy\n \nelif set(list_features) == set([ 'T5_43k', 'E2560_43k' , 'taxons31_onehot_43k']):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t36B3S2560_train_embeds_float32_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),], axis =1)\n\n # dict_data_cfg5 = {'list_features_ids': [ 'T5_43k','E2560_43k' , 'taxons31_onehot_43k'] }\n # /kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t36B3S2560_train_embeds_float32_cut43k.npy\n\nelif set(list_features) == set([ 'T5_43k','E320_43k' , 'E480_43k' , 'taxons31_onehot_43k']):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t6M8S320_train_embeds_float32_cut43k.npy'\n fn3 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t12M35S480_train_embeds_float32_cut43k.npy'\n fn4 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), np.load(fn3),\n pd.read_csv(fn4,index_col = 0).values.astype(np.float32), ], axis =1)\n \n #list_features = ['T5_43k', 'E320_43k' , 'E480_43k' , 'taxons31_onehot_43k' ] # , 'F06_genes' # ['T5_43k']# \n # /kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t6M8S320_train_embeds_float32_cut43k.npy\n # /kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t12M35S480_train_embeds_float32_cut43k.npy\n\nelif set(list_features) == set([ 'T5_43k', 'E2560mean_43k' , 'taxons31_onehot_43k']):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t36B3S2560mean_train_float32_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),], axis =1)\n \nelif set(list_features) == set([ 'T5_43k', 'E1280_43k' , 'taxons31_onehot_43k']):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t33M650S1280_train_embeds_float32_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),], axis =1)\nelif set(list_features) == set([ 'T5_43k', 'protb_43k' , 'taxons31_onehot_43k']):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_protb_train_float32_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),], axis =1)\nelif set(list_features) == set([ 'T5_43k', 'E5120_43k' , 'taxons31_onehot_43k']):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t48B15S5120_train_float32_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),], axis =1)\nelif set(list_features) == set([ 'E5120_43k' , 'taxons31_onehot_43k']):\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t48B15S5120_train_float32_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n X = np.concatenate( [ np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),], axis =1)\nelif set(list_features) == set([ 'E5120_43k']):\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t48B15S5120_train_float32_cut43k.npy'\n X = np.load(fn2)\n\n\nX = X[:n_samples_to_consider,:]\nprint('X.shape', X.shape, 'effective') \nprint('type(X[0,0])' , type(X[0,0]) )\nprint(X[:3,:2] , X.sum() )","metadata":{"papermill":{"duration":6.255988,"end_time":"2023-07-15T21:44:53.514595","exception":false,"start_time":"2023-07-15T21:44:47.258607","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-20T20:34:54.108975Z","iopub.execute_input":"2023-07-20T20:34:54.110053Z","iopub.status.idle":"2023-07-20T20:34:57.131255Z","shell.execute_reply.started":"2023-07-20T20:34:54.109989Z","shell.execute_reply":"2023-07-20T20:34:57.129891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \nv = Y.sum(axis = 0)\nplt.plot(v)\nplt.show()\npd.Series(v).describe()\n","metadata":{"papermill":{"duration":0.483228,"end_time":"2023-07-15T21:44:54.012916","exception":false,"start_time":"2023-07-15T21:44:53.529688","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-20T20:34:57.134457Z","iopub.execute_input":"2023-07-20T20:34:57.134942Z","iopub.status.idle":"2023-07-20T20:34:57.650102Z","shell.execute_reply.started":"2023-07-20T20:34:57.1349Z","shell.execute_reply":"2023-07-20T20:34:57.648637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n######################################################################################3\n############### Load trainTerms\n######################################################################################3\n\nprint()\nfn = '/kaggle/input/cafa-5-protein-function-prediction/Train/train_terms.tsv'\nprint(fn)\ntrainTerms = pd.read_csv(fn, sep=\"\\t\")\nprint(trainTerms.shape)\ndisplay(trainTerms.head(3))\n\n","metadata":{"papermill":{"duration":3.422018,"end_time":"2023-07-15T21:44:57.451216","exception":false,"start_time":"2023-07-15T21:44:54.029198","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-20T20:34:57.651925Z","iopub.execute_input":"2023-07-20T20:34:57.652379Z","iopub.status.idle":"2023-07-20T20:35:00.745906Z","shell.execute_reply.started":"2023-07-20T20:34:57.65233Z","shell.execute_reply":"2023-07-20T20:35:00.744427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if (mode_submit is not None) and ( mode_submit != False ): # None\n\n X_submit = np.zeros( (141865,0), dtype = np.float32 )\n if 'T5' in str(list_features) :\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_submit_T5_float32.npy'\n print(fn)\n X_submit = np.concatenate( [X_submit, np.load(fn)], axis =1)\n if 'esm2Fairushchin' in str(list_features) :\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_submit_esm2Fairushchin_float32_orderedAsInput.npy'\n print(fn)\n X_submit = np.concatenate( [X_submit, np.load(fn)], axis =1)\n\n # fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_submit_T5_float32.npy'\n # fn2 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_submit_esm2Fairushchin_float32_orderedAsInput.npy'\n # X_submit = np.concatenate( [np.load(fn), np.load(fn2)], axis =1)\n if 'taxons31_onehot' in str(list_features): \n fn = '/kaggle/input/cafa5-data-selected/features/test_features05_taxons31_onehot.csv'\n print(fn)\n X_submit = np.concatenate( [X_submit, pd.read_csv(fn,index_col = 0).values.astype(np.float32)], axis =1)\n if 'F06_genes' in str(list_features): \n fn = '/kaggle/input/cafa5-data-selected/features/test_features06_genes_names_related_onehot.csv'\n print(fn)\n X_submit = np.concatenate( [X_submit, pd.read_csv(fn,index_col = 0).values.astype(np.float32)], axis =1)\n if 'F07_DesriptKw1H' in str(list_features): \n fn = '/kaggle/input/cafa5-data-selected/features/test_features07_keywords_onehot.csv'\n print(fn)\n X_submit = np.concatenate( [X_submit, pd.read_csv(fn,index_col = 0).values.astype(np.float32)], axis =1)\n \n \n print( X_submit.shape, type(X[0,0]) )\n print( X_submit[:3,:2] )\n\n","metadata":{"papermill":{"duration":0.034102,"end_time":"2023-07-15T21:44:57.501407","exception":false,"start_time":"2023-07-15T21:44:57.467305","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-20T20:35:00.748241Z","iopub.execute_input":"2023-07-20T20:35:00.750117Z","iopub.status.idle":"2023-07-20T20:35:00.772881Z","shell.execute_reply.started":"2023-07-20T20:35:00.750051Z","shell.execute_reply":"2023-07-20T20:35:00.771136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Metric code","metadata":{"papermill":{"duration":0.015587,"end_time":"2023-07-15T21:44:57.532587","exception":false,"start_time":"2023-07-15T21:44:57.517","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Evaluation for CAFA \n# https://github.com/BioComputingUP/CAFA-evaluator\n\nimport numpy as np\nimport pandas as pd\nimport multiprocessing as mp\nimport copy\nimport logging\n\nclass Graph:\n \"\"\"\n Ontology class. One ontology == one namespace\n DAG is the adjacence matrix (sparse) which represent a Directed Acyclic Graph where\n DAG(i,j) == 1 means that the go term i is_a (or is part_of) j\n Parents that are in a different namespace are discarded\n \"\"\"\n def __init__(self, namespace, terms_dict, ia_dict=None, orphans=False):\n \"\"\"\n terms_dict = {term: {name: , namespace: , def: , alt_id: , rel:}}\n \"\"\"\n self.namespace = namespace\n self.dag = [] # [[], ...] terms (rows, axis 0) x parents (columns, axis 1)\n self.terms_dict = {} # {term: {index: , name: , namespace: , def: } used to assign term indexes in the gt\n self.terms_list = [] # [{id: term, name:, namespace: , def:, adg: [], children: []}, ...]\n self.idxs = None # Number of terms\n self.order = None\n self.toi = None\n self.ia = None\n\n rel_list = []\n for self.idxs, (term_id, term) in enumerate(terms_dict.items()):\n rel_list.extend([[term_id, rel, term['namespace']] for rel in term['rel']])\n self.terms_list.append({'id': term_id, 'name': term['name'], 'namespace': namespace, 'def': term['def'],\n 'adj': [], 'children': []})\n self.terms_dict[term_id] = {'index': self.idxs, 'name': term['name'], 'namespace': namespace, 'def': term['def']}\n for a_id in term['alt_id']:\n self.terms_dict[a_id] = copy.copy(self.terms_dict[term_id])\n self.idxs += 1\n\n self.dag = np.zeros((self.idxs, self.idxs), dtype='bool')\n\n # id1 term (row, axis 0), id2 parent (column, axis 1)\n for id1, id2, ns in rel_list:\n if self.terms_dict.get(id2):\n i = self.terms_dict[id1]['index']\n j = self.terms_dict[id2]['index']\n self.dag[i, j] = 1\n self.terms_list[i]['adj'].append(j)\n self.terms_list[j]['children'].append(i)\n logging.debug(\"i,j {},{} {},{}\".format(i, j, id1, id2))\n else:\n logging.debug('Skipping branch to external namespace: {}'.format(id2))\n logging.debug(\"dag {}\".format(self.dag))\n # Topological sorting\n self.top_sort()\n logging.debug(\"order sorted {}\".format(self.order))\n\n if orphans:\n self.toi = np.arange(self.dag.shape[0]) # All terms, also those without parents\n else:\n self.toi = np.nonzero(self.dag.sum(axis=1) > 0)[0] # Only terms with parents\n logging.debug(\"toi {}\".format(self.toi))\n\n if ia_dict is not None:\n self.set_ia(ia_dict)\n\n return\n\n def top_sort(self):\n \"\"\"\n Takes a sparse matrix representing a DAG and returns an array with nodes indexes in topological order\n https://en.wikipedia.org/wiki/Topological_sorting\n \"\"\"\n indexes = []\n visited = 0\n (rows, cols) = self.dag.shape\n\n # create a vector containing the in-degree of each node\n in_degree = self.dag.sum(axis=0)\n # logging.debug(\"degree {}\".format(in_degree))\n\n # find the nodes with in-degree 0 (leaves) and add them to the queue\n queue = np.nonzero(in_degree == 0)[0].tolist()\n # logging.debug(\"queue {}\".format(queue))\n\n # for each element of the queue increment visits, add them to the list of ordered nodes\n # and decrease the in-degree of the neighbor nodes\n # and add them to the queue if they reach in-degree == 0\n while queue:\n visited += 1\n idx = queue.pop(0)\n indexes.append(idx)\n in_degree[idx] -= 1\n l = self.terms_list[idx]['adj']\n if len(l) > 0:\n for j in l:\n in_degree[j] -= 1\n if in_degree[j] == 0:\n queue.append(j)\n\n # if visited is equal to the number of nodes in the graph then the sorting is complete\n # otherwise the graph can't be sorted with topological order\n if visited == rows:\n self.order = indexes\n else:\n raise Exception(\"The sparse matrix doesn't represent an acyclic graph\")\n\n def set_ia(self, ia_dict):\n self.ia = np.zeros(self.idxs, dtype='float')\n for term_id in self.terms_dict:\n if ia_dict.get(term_id):\n self.ia[self.terms_dict[term_id]['index']] = ia_dict.get(term_id)\n else:\n logging.debug('Missing IA for term: {}'.format(term_id))\n # Convert inf to zero\n np.nan_to_num(self.ia, copy=False, nan=0, posinf=0, neginf=0)\n self.toi = np.nonzero(self.ia > 0)[0]\n\n\nclass Prediction:\n \"\"\"\n The score matrix contains the scores given by the predictor for every node of the ontology\n \"\"\"\n def __init__(self, ids, matrix, idx, namespace=None):\n self.ids = ids\n self.matrix = matrix # scores\n self.next_idx = idx\n # self.n_pred_seq = idx + 1\n self.namespace = namespace\n\n def __str__(self):\n return \"\\n\".join([\"{}\\t{}\\t{}\".format(index, self.matrix[index], self.namespace) for index, _id in enumerate(self.ids)])\n\n\nclass GroundTruth:\n def __init__(self, ids, matrix, namespace=None):\n self.ids = ids\n self.matrix = matrix\n self.namespace = namespace\n\n\ndef propagate(matrix, ont, order, mode='max'):\n \"\"\"\n Update inplace the score matrix (proteins x terms) up to the root taking the max between children and parents\n \"\"\"\n if matrix.shape[0] == 0:\n raise Exception(\"Empty matrix\")\n\n deepest = np.where(np.sum(matrix[:, order], axis=0) > 0)[0][0]\n if deepest.size == 0:\n raise Exception(\"The matrix is empty\")\n\n # Remove leaves\n order_ = np.delete(order, [range(0, deepest)])\n\n for i in order_:\n # Get direct children\n children = np.where(ont.dag[:, i] != 0)[0]\n if children.size > 0:\n cols = np.concatenate((children, [i]))\n if mode == 'max':\n matrix[:, i] = matrix[:, cols].max(axis=1)\n elif mode == 'fill':\n rows = np.where(matrix[:, i] == 0)[0]\n if rows.size:\n idx = np.ix_(rows, cols)\n if flag_correct_metric_computation_bug_found_by_Anton:\n # Corrected way (see https://www.kaggle.com/competitions/cafa-5-protein-function-prediction/discussion/420241 )\n matrix[rows, i] = matrix[idx].max(axis=1) # matrix[idx].max(axis=1)[0] # Correction: https://www.kaggle.com/competitions/cafa-5-protein-function-prediction/discussion/420241\n else:\n # Old way - not corrected\n matrix[rows, i] = matrix[idx].max(axis=1)[0] # Correction: https://www.kaggle.com/competitions/cafa-5-protein-function-prediction/discussion/420241\n \n return\n\n\ndef obo_parser(obo_file, valid_rel=(\"is_a\", \"part_of\")):\n \"\"\"\n Parse a OBO file and returns a list of ontologies, one for each namespace.\n Obsolete terms are excluded as well as external namespaces.\n \"\"\"\n term_dict = {}\n term_id = None\n namespace = None\n name = None\n term_def = None\n alt_id = []\n rel = []\n obsolete = True\n with open(obo_file) as f:\n for line in f:\n line = line.strip().split(\": \")\n if line and len(line) > 1:\n k = line[0]\n v = \": \".join(line[1:])\n if k == \"id\":\n # Populate the dictionary with the previous entry\n if term_id is not None and obsolete is False and namespace is not None:\n term_dict.setdefault(namespace, {})[term_id] = {'name': name,\n 'namespace': namespace,\n 'def': term_def,\n 'alt_id': alt_id,\n 'rel': rel}\n # Assign current term ID\n term_id = v\n\n # Reset optional fields\n alt_id = []\n rel = []\n obsolete = False\n namespace = None\n\n elif k == \"alt_id\":\n alt_id.append(v)\n elif k == \"name\":\n name = v\n elif k == \"namespace\" and v != 'external':\n namespace = v\n elif k == \"def\":\n term_def = v\n elif k == 'is_obsolete':\n obsolete = True\n elif k == \"is_a\" and k in valid_rel:\n s = v.split('!')[0].strip()\n rel.append(s)\n elif k == \"relationship\" and v.startswith(\"part_of\") and \"part_of\" in valid_rel:\n s = v.split()[1].strip()\n rel.append(s)\n\n # Last record\n if obsolete is False and namespace is not None:\n term_dict.setdefault(namespace, {})[term_id] = {'name': name,\n 'namespace': namespace,\n 'def': term_def,\n 'alt_id': alt_id,\n 'rel': rel}\n return term_dict\n\n\ndef gt_parser(gt_file, ontologies):\n \"\"\"\n Parse ground truth file. Discard terms not included in the ontology.\n \"\"\"\n gt_dict = {}\n with open(gt_file) as f:\n for line in f:\n line = line.strip().split()\n if line:\n p_id, term_id = line[:2]\n for ont in ontologies:\n if term_id in ont.terms_dict:\n gt_dict.setdefault(ont.namespace, {}).setdefault(p_id, []).append(term_id)\n break\n\n gts = {}\n for ont in ontologies:\n if gt_dict.get(ont.namespace):\n matrix = np.zeros((len(gt_dict[ont.namespace]), ont.idxs), dtype='bool')\n ids = {}\n for i, p_id in enumerate(gt_dict[ont.namespace]):\n ids[p_id] = i\n for term_id in gt_dict[ont.namespace][p_id]:\n matrix[i, ont.terms_dict[term_id]['index']] = 1\n propagate(matrix, ont, ont.order, mode='max')\n gts[ont.namespace] = GroundTruth(ids, matrix, ont.namespace)\n\n return gts\n\n\ndef pred_parser(f, ontologies, gts, prop_mode, max_terms=None):\n \"\"\"\n Parse a prediction file and returns a list of prediction objects, one for each namespace.\n If a predicted is predicted multiple times for the same target, it stores the max.\n This is the slow step if the input file is huge, ca. 1 minute for 5GB input on SSD disk.\n \"\"\"\n ids = {}\n matrix = {}\n ns_dict = {} # {namespace: term}\n onts = {ont.namespace: ont for ont in ontologies}\n for ns in gts:\n matrix[ns] = np.zeros(gts[ns].matrix.shape, dtype='float')\n ids[ns] = {}\n for term in onts[ns].terms_dict:\n ns_dict[term] = ns\n\n for line in f:\n p_id, term_id, prob = line\n ns = ns_dict.get(term_id)\n if ns in gts and p_id in gts[ns].ids:\n i = gts[ns].ids[p_id]\n if max_terms is None or np.count_nonzero(matrix[ns][i]) <= max_terms:\n j = onts[ns].terms_dict.get(term_id)['index']\n ids[ns][p_id] = i\n matrix[ns][i, j] = max(matrix[ns][i, j], float(prob))\n\n predictions = []\n for ns in ids:\n if ids[ns]:\n propagate(matrix[ns], onts[ns], onts[ns].order, mode=prop_mode)\n predictions.append(Prediction(ids[ns], matrix[ns], len(ids[ns]), ns))\n\n if not predictions:\n raise Exception(\"Empty prediction, check format\")\n\n return predictions\n\n\ndef ia_parser(file):\n ia_dict = {}\n with open(file) as f:\n for line in f:\n if line:\n term, ia = line.strip().split()\n ia_dict[term] = float(ia)\n return ia_dict\n\n# Computes the root terms in the dag\ndef get_roots_idx(dag):\n return np.where(dag.sum(axis=1) == 0)[0]\n\n\n# Computes the leaf terms in the dag\ndef get_leafs_idx(dag):\n return np.where(dag.sum(axis=0) == 0)[0]\n\n\n# Return a mask for all the predictions (matrix) >= tau\ndef solidify_prediction(pred, tau):\n return pred >= tau\n\n\n# computes the f metric for each precision and recall in the input arrays\ndef compute_f(pr, rc):\n n = 2 * pr * rc\n d = pr + rc\n return np.divide(n, d, out=np.zeros_like(n, dtype=float), where=d != 0)\n\n\ndef compute_s(ru, mi):\n return np.sqrt(ru**2 + mi**2)\n # return np.where(np.isnan(ru), mi, np.sqrt(ru + np.nan_to_num(mi)))\n\nimport time\nfrom scipy.sparse import csr_matrix\n\ndef compute_metrics_(tau_arr, g, pred, toi, n_gt, wn_gt=None, ic_arr=None):\n\n verbose = 0; \n\n if verbose >= 10:\n t0 = time.time()\n \n metrics = np.zeros((len(tau_arr), 7), dtype='float') # cov, pr, rc, wpr, wrc, ru, mi\n\n if verbose >= 10:\n print('type(toi), toi', type(toi), toi )\n tmp = pred.matrix[:, toi]\n if verbose >= 10:\n print('type(tmp), tmp.shape', type(tmp), tmp.shape )\n p_s = csr_matrix(tmp )\n ic_arr_toi = ic_arr[toi]\n if verbose >= 10:\n print('type(ic_arr_toi), ic_arr_toi.shape', type(ic_arr_toi), ic_arr_toi.shape )\n\n \n g_s = csr_matrix( g )\n \n if verbose >= 10:\n print( 'csr_matrix done %.1f'%(time.time( ) - t0 ), 'p_s.shape, g_s.shape:', p_s.shape, g_s.shape ) \n\n\n \n for i, tau in enumerate(tau_arr):\n\n# p = solidify_prediction(pred_matrix_toi, tau)\n\n# # number of proteins with at least one term predicted with score >= tau\n# metrics[i, 0] = (p.sum(axis=1) > 0).sum()\n\n# # Terms subsets\n# intersection = np.logical_and(p, g) # TP # SLOW PART !!!\n\n# # Subsets size\n# n_pred = p.sum(axis=1)\n# n_intersection = intersection.sum(axis=1)\n\n# # Precision, recall\n# metrics[i, 1] = np.divide(n_intersection, n_pred, out=np.zeros_like(n_intersection, dtype='float'),\n# where=n_pred > 0).sum()\n# metrics[i, 2] = np.divide(n_intersection, n_gt, out=np.zeros_like(n_gt, dtype='float'), where=n_gt > 0).sum()\n\n\n if verbose >= 100:\n t0 = time.time()\n print()\n print(i, tau, 'Start %.1f'%(time.time( ) - t0 ) )\n p = p_s > tau # solidify_prediction(p, tau)\n if verbose >= 100:\n print(i, tau, 'solidify done %.1f'%(time.time( ) - t0 ), 'p.shape:', p.shape, )\n# p_s = csr_matrix(p)\n# print(i, tau, 'csr_matrix done %.1f'%(time.time( ) - t0 ), 'p.shape:', p.shape, )\n \n# print(p.shape)\n # number of proteins with at least one term predicted with score >= tau\n metrics[i, 0] = (p.sum(axis=1) > 0).sum()\n\n # Terms subsets\n# intersection = np.logical_and(p, g) # TP\n intersection = p.multiply( g_s) # TP\n\n\n if ic_arr is not None:\n \n # Weighted precision, recall\n # wn_pred = (p * ic_arr_toi).sum(axis=1) # \n# wn_pred = np.dot(p , ic_arr_toi)\n wn_pred = p.dot( ic_arr_toi)\n # wn_intersection = (intersection *ic_arr_toi).sum(axis=1)\n# wn_intersection = np.dot( intersection , ic_arr_toi )\n wn_intersection = intersection.dot( ic_arr_toi )\n \n if verbose >= 100:\n print(i, tau, 'After w_pred wn_intersection %.1f'%(time.time( ) - t0 ) )\n \n metrics[i, 3] = np.divide(wn_intersection, wn_pred, out=np.zeros( wn_intersection.shape, dtype='float'),\n where=wn_pred > 0).sum()\n metrics[i, 4] = np.divide(wn_intersection, wn_gt, out=np.zeros(wn_intersection.shape, dtype='float'),\n where=n_gt > 0).sum()\n if verbose >= 100:\n print(i, tau, 'After metrics 3,4 %.1f'%(time.time( ) - t0 ) )\n\n# # Terms subsets\n# remaining = np.logical_and(np.logical_not(p), g) # FN --> not predicted but in the ground truth\n# mis = np.logical_and(p, np.logical_not(g)) # FP --> predicted but not in the ground truth\n\n# print(i, tau, 'After remining and miss %.1f'%(time.time( ) - t0 ) )\n \n# # Misinformation, remaining uncertainty\n# metrics[i, 5] = (remaining * ic_arr_toi).sum(axis=1).sum()\n# metrics[i, 6] = (mis * ic_arr_toi).sum(axis=1).sum()\n\n return metrics\n\n\ndef compute_metrics(pred, gt, toi, tau_arr, ic_arr=None, n_cpu=0):\n \"\"\"\n Takes the prediction and the ground truth and for each threshold in tau_arr\n calculates the confusion matrix and returns the coverage,\n precision, recall, remaining uncertainty and misinformation.\n Toi is the list of terms (indexes) to be considered\n \"\"\"\n g = gt.matrix[:, toi]\n n_gt = g.sum(axis=1)\n wn_gt = None\n if ic_arr is not None:\n wn_gt = (g * ic_arr[toi]).sum(axis=1)\n\n # Parallelization\n if n_cpu == 0:\n n_cpu = mp.cpu_count()\n\n arg_lists = [[tau_arr, g, pred, toi, n_gt, wn_gt, ic_arr] for tau_arr in np.array_split(tau_arr, n_cpu)]\n if 0:\n # Original parallel way (# It does not work on Kaggle)\n arg_lists = [[tau_arr, g, pred, toi, n_gt, wn_gt, ic_arr] for tau_arr in np.array_split(tau_arr, n_cpu)]\n with mp.Pool(processes=n_cpu) as pool:\n metrics = np.concatenate(pool.starmap(compute_metrics_, arg_lists), axis=0)\n else: \n # no-parallel: \n metrics = compute_metrics_(tau_arr, g, pred, toi, n_gt, wn_gt, ic_arr )\n\n return pd.DataFrame(metrics, columns=[\"cov\", \"pr\", \"rc\", \"wpr\", \"wrc\", \"ru\", \"mi\"])\n\n\ndef evaluate_prediction(prediction, gt, ontologies, tau_arr, normalization='cafa', n_cpu=0):\n dfs = []\n for p in prediction:\n ns = p.namespace\n ne = np.full(len(tau_arr), gt[ns].matrix.shape[0])\n\n ont = [o for o in ontologies if o.namespace == ns][0]\n\n # cov, pr, rc, wpr, wrc, ru, mi\n metrics = compute_metrics(p, gt[ns], ont.toi, tau_arr, ont.ia, n_cpu)\n\n for column in [\"pr\", \"rc\", \"wpr\", \"wrc\", \"ru\", \"mi\"]:\n if normalization == 'gt' or (column in [\"rc\", \"wrc\"] and normalization == 'cafa'):\n metrics[column] = np.divide(metrics[column], ne, out=np.zeros_like(metrics[column], dtype='float'), where=ne > 0)\n else:\n metrics[column] = np.divide(metrics[column], metrics[\"cov\"], out=np.zeros_like(metrics[column], dtype='float'), where=metrics[\"cov\"] > 0)\n\n metrics['ns'] = [ns] * len(tau_arr)\n metrics['tau'] = tau_arr\n metrics['cov'] = np.divide(metrics['cov'], ne, out=np.zeros_like(metrics['cov'], dtype='float'), where=ne > 0)\n metrics['f'] = compute_f(metrics['pr'], metrics['rc'])\n metrics['wf'] = compute_f(metrics['wpr'], metrics['wrc'])\n metrics['s'] = compute_s(metrics['ru'], metrics['mi'])\n\n dfs.append(metrics)\n\n return pd.concat(dfs)","metadata":{"papermill":{"duration":0.117015,"end_time":"2023-07-15T21:44:57.665237","exception":false,"start_time":"2023-07-15T21:44:57.548222","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-20T20:35:00.778951Z","iopub.execute_input":"2023-07-20T20:35:00.780123Z","iopub.status.idle":"2023-07-20T20:35:00.912264Z","shell.execute_reply.started":"2023-07-20T20:35:00.780034Z","shell.execute_reply":"2023-07-20T20:35:00.910493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n# Tau array, used to compute metrics at different score thresholds\nth_step = 0.01\ntau_arr = np.arange(0.01, 1, th_step)\n#Consider terms without parents, e.g. the root(s), in the evaluation\nno_orphans = False\n# Parse and set information accretion (optional)\nia_dict = ia_parser('/kaggle/input/cafa-5-protein-function-prediction/IA.txt')\n\n# Parse the OBO file and creates a different graph for each namespace\nontologies = []\nobo_file = '/kaggle/input/cafa-5-protein-function-prediction/Train/go-basic.obo'\nfor ns, terms_dict in obo_parser(obo_file).items():\n ontologies.append(Graph(ns, terms_dict, ia_dict, not no_orphans))","metadata":{"papermill":{"duration":3.044506,"end_time":"2023-07-15T21:45:00.725572","exception":false,"start_time":"2023-07-15T21:44:57.681066","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-20T20:35:00.915207Z","iopub.execute_input":"2023-07-20T20:35:00.915856Z","iopub.status.idle":"2023-07-20T20:35:04.640221Z","shell.execute_reply.started":"2023-07-20T20:35:00.915811Z","shell.execute_reply":"2023-07-20T20:35:04.638855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling preparations","metadata":{"papermill":{"duration":0.015917,"end_time":"2023-07-15T21:45:00.75802","exception":false,"start_time":"2023-07-15T21:45:00.742103","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\nimport os.path\n\ndef get_F1_etc_scores_official_CAFA_evaluation( Y_pred, IX, cutoff_threshold_low = 0.01, make_plots = True , verbose = 0 ): \n # Input: \n '''\n Computation of F1-weighted scores are called here. \n Here we prepare Y_pred, Y in format required by functions provided by organizers - see github: https://github.com/BioComputingUP/CAFA-evaluator\n Y_pred - predictions\n trainTerms # initial file with training labels \n IX - indices selecting part which correspond to Y_pred in full Y \n Params:\n cutoff_threshold_low - predictions lower (strictly) will be dropped (effectively set to zero)\n make_plots = True \n verbose = 0\n \n Function uses external variables: \n trainTerms - training labels provided by orgs: /kaggle/input/cafa-5-protein-function-prediction/Train/train_terms.tsv\n vec_train_protein_ids - ids of the proteins in the current train - should correspond to \"X\" - features \n ontologies - data from: /kaggle/input/cafa-5-protein-function-prediction/Train/go-basic.obo\n tau_arr - array of thresholds \n '''\n\n # Params:\n\n t00 = time.time()\n if verbose >= 100:\n print('Scoring starts. n_samples:', len(IX) )\n\n ##########################################################################################\n # Prepare \"ground truth\" - \"gt\" terms(labels) in required format \n ##########################################################################################\n\n # First save to file, because function \"gt_parser\" works with files as input \n # Only part corresponding to providex indices IX will be generated \n t0 = time.time()\n trainTerms[ trainTerms.EntryID.isin(vec_train_protein_ids[IX]) ].to_csv('valid.tsv', sep='\\t', index=False) # Wall time: 4.11 s for 28k samples\n if verbose >= 1000:\n print('save valid.csv %.1f'%(time.time() - t0 )) \n\n # Prepare \"gt\" labels \n t0 = time.time()\n gt = gt_parser('valid.tsv', ontologies) # Wall time: 1min 22s for 28k samples\n if verbose >= 100:\n print('gt_parser %.1f'%(time.time() - t0 ))\n\n ##########################################################################################\n # prepare predicitons as list of triples - (protein, term(label), prediction) \n ##########################################################################################\n\n t0 = time.time()\n vec_train_protein_ids_loc = vec_train_protein_ids[IX]\n preds = []\n for i in range(len(vec_train_protein_ids_loc)):\n for j in range(len(labels_to_consider)):\n if Y_pred[i,j] >= cutoff_threshold_low:\n preds.append((vec_train_protein_ids_loc[i], \n labels_to_consider[j],\n Y_pred[i,j] ))\n if verbose >= 1000: \n print('create preds %.1f'%(time.time() - t0 )) \n\n ##########################################################################################\n # Parse predictions - propagation happens here \n ##########################################################################################\n t0 = time.time()\n preds = pred_parser(preds, ontologies, gt, prop_mode='fill', max_terms=500) # \n if verbose >= 1000: \n print('pred_parser %.1f'%(time.time() - t0 ), 'len(preds)', len(preds) ) \n\n gc.collect()\n\n ##########################################################################################\n # Main scores calculations happends here: \n ##########################################################################################\n # %%time\n t0 = time.time()\n df_metrics = evaluate_prediction(preds, gt, ontologies, tau_arr, n_cpu=1) # Wall time: 37.7 s for 28k samples\n if verbose >= 1000: \n print('evaluate_prediction %.1f'%(time.time() - t0 ), 'got df_metrics with shape:', df_metrics.shape ) \n if verbose >= 10000: \n display( df_metrics.head(2) )\n\n ##########################################################################################\n # Comptutations finished. Below are optional plots, output preparartions etc. \n ##########################################################################################\n if verbose >= 100:\n _t = df_metrics.groupby('ns').agg({'wf':'max'})\n display( _t )\n print( _t.mean() ) \n\n if verbose >= 100:\n print('F1-scoring finished. %.1f secs passed'%(time.time() - t00 ))\n\n # %%time\n if make_plots:\n try:\n list_uv = list(df_metrics['ns'].unique() )\n #print(list_uv)\n fig = plt.figure(figsize = (20,4))\n i0 = 0;\n for col in ['wf', 'wpr', 'wrc' ] :\n i0+=1\n # print(i0,col)\n fig.add_subplot(1,3,i0)\n\n for uv in list_uv:\n mask = df_metrics['ns'] == uv\n v = df_metrics[mask][col]\n plt.plot(v.values, label = uv)\n plt.title(col, fontsize = 20)\n plt.legend()\n plt.grid()\n plt.show() \n except:\n print('Exception in plot')\n \n ########################################################################################\n # Prepare output of scores : \n ########################################################################################\n _t = {'cellular_component':'CCO', 'biological_process':'BPO','molecular_function':'MFO'}\n dict_scores_etc = {}\n df_s = df_metrics.groupby('ns').agg({'wf':'max'})\n dict_scores_etc['F1w'] = np.round( df_s.mean().iloc[0], 6) \n for k in _t:\n k2 = _t[k]\n # print(k,dict_scores_etc )\n if k in df_s.index:\n dict_scores_etc['F1 '+ k2 ] = np.round( df_s.loc[k].iat[0], 6) \n else:\n dict_scores_etc['F1 '+ k2 ] = 0\n\n ########################################################################################\n # Prepare output of thresholds : \n ########################################################################################\n for k in _t:\n k2 = _t[k]\n m = df_metrics['ns'] == k\n if m.sum()>0:\n IX = df_metrics[m]['wf'].argmax()\n thres_optimal = df_metrics[m]['tau'].iat[IX]\n dict_scores_etc['thres '+ k2 ] = thres_optimal\n else:\n dict_scores_etc['thres '+ k2 ] = 0\n \n dict_scores_etc['F-Scores Time'] = np.round( time.time() - t00 ,1) \n if verbose >= 100:\n print('Scores: ', dict_scores_etc) \n\n if os.path.isfile('valid.tsv') :\n os.remove('valid.tsv')\n \n return dict_scores_etc ","metadata":{"papermill":{"duration":0.043082,"end_time":"2023-07-15T21:45:00.817069","exception":false,"start_time":"2023-07-15T21:45:00.773987","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-20T20:35:04.642333Z","iopub.execute_input":"2023-07-20T20:35:04.643242Z","iopub.status.idle":"2023-07-20T20:35:04.669372Z","shell.execute_reply.started":"2023-07-20T20:35:04.643196Z","shell.execute_reply":"2023-07-20T20:35:04.667995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndef update_scores(df_stat, str_full_id, fold_id, Y , Y_pred,IX, cutoff_threshold_low = 0.01, fit_time = None, predict_time = None , blend_mode = None ,\n str_cv_info = None ):\n IX_df_stat = len(df_stat) + 1 \n t0 = time.time()\n df_stat.loc[IX_df_stat,'FullId'] = str_full_id\n try: \n # for tpu enviroment we do not have sklearn pre-install\n df_stat.loc[IX_df_stat,'RocAuc ravel'] = np.round( roc_auc_score (Y.ravel(),Y_pred.ravel() ) , 4)\n except:\n pass\n\n dict_scores_etc = get_F1_etc_scores_official_CAFA_evaluation( Y_pred, IX, cutoff_threshold_low = cutoff_threshold_low , make_plots = True , verbose = 10000 )\n for k in dict_scores_etc:\n df_stat.loc[IX_df_stat,k] = dict_scores_etc[k]\n \n for t in [0.2, 0.3,0.4,0.5]:\n _c = (Y_pred >= t).ravel().sum()\n df_stat.loc[IX_df_stat,'GE%.1f per prot'%(t)] = np.round(_c/Y.shape[0])\n \n \n df_stat.loc[IX_df_stat,'Fold'] = fold_id\n\n df_stat.loc[IX_df_stat,'Scoring time'] = np.round( time.time() - t0,1 )\n df_stat.loc[IX_df_stat,'CV'] = str_cv_info\n df_stat.loc[IX_df_stat,'Fit time'] = np.round( fit_time , 1)\n df_stat.loc[IX_df_stat,'Predict time'] = np.round( predict_time , 1)\n df_stat.loc[IX_df_stat,'n_targets'] = Y.shape[1]\n df_stat.loc[IX_df_stat,'n_samples'] = Y.shape[0]\n df_stat.loc[IX_df_stat,'blend_mode'] = blend_mode\n \n display(df_stat.tail(5))\n df_stat.to_csv('df_foldwise_stat.csv')\n","metadata":{"papermill":{"duration":0.031564,"end_time":"2023-07-15T21:45:00.864727","exception":false,"start_time":"2023-07-15T21:45:00.833163","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-20T20:35:04.671516Z","iopub.execute_input":"2023-07-20T20:35:04.671969Z","iopub.status.idle":"2023-07-20T20:35:04.694003Z","shell.execute_reply.started":"2023-07-20T20:35:04.671922Z","shell.execute_reply":"2023-07-20T20:35:04.692231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nimport gc\ntry: \n from sklearn.metrics import roc_auc_score\n from sklearn.metrics import f1_score\n from sklearn.linear_model import Ridge\n from sklearn.neural_network import MLPRegressor\n import lightgbm as lgbm\n from sklearn.multioutput import MultiOutputRegressor\n from sklearn.multioutput import MultiOutputClassifier \n \n from catboost import CatBoostRegressor\n from catboost import CatBoostClassifier \nexcept:\n print('Exception importing models ')\n pass\n\nimport time \n\n\n# Use Keras or tensorflow.keras, don't use both of them.\n# https://stackoverflow.com/questions/63761504/typeerror-the-added-layer-must-be-an-instance-of-class-layer-found-keras-lay\n\n# from tensorflow import keras\n# from tensorflow.keras.models import Sequential\n# from tensorflow.keras.layers import Dense\n# from tensorflow.keras.layers import Dropout\n# from tensorflow.keras.layers import BatchNormalization\n# from tensorflow.keras.metrics import AUC\n\nimport keras\nfrom keras.models import Sequential, Model\nfrom keras.layers import Dense, Input, Concatenate, Dropout, BatchNormalization, Activation\nfrom keras.layers import LayerNormalization\n\nfrom keras.metrics import AUC\nfrom keras.utils import plot_model\nfrom keras.optimizers import RMSprop, Adam, SGD\n\n","metadata":{"papermill":{"duration":6.857254,"end_time":"2023-07-15T21:45:08.021183","exception":false,"start_time":"2023-07-15T21:45:01.163929","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-20T20:35:04.695981Z","iopub.execute_input":"2023-07-20T20:35:04.696415Z","iopub.status.idle":"2023-07-20T20:35:04.717647Z","shell.execute_reply.started":"2023-07-20T20:35:04.696361Z","shell.execute_reply":"2023-07-20T20:35:04.715641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n\ndef get_model( config_model_features_postprocessing_etc , verbose = 100 ): \n '''\n config_model_features_postprocessing_etc - dictionary with configurations\n '''\n # See also : https://www.kaggle.com/code/alexandervc/cafa5-mlp-tune-1#Get-model-and-other-auxiliary-functions\n \n model_cfg = config_model_features_postprocessing_etc['Model']\n \n if verbose >= 100:\n print( model_cfg )\n \n if model_cfg[0] == 'Ridge':\n alpha = model_cfg[1]['alpha'];\n str_model_id = 'Ridge'+str(alpha)\n model = Ridge(alpha=alpha)\n elif model_cfg[0] == 'gpuLogReg':\n import torch\n if torch.cuda.is_available():\n from cuml.linear_model import LogisticRegression as CULogisticRegression \n model = MultiOutputClassifier(estimator= CULogisticRegression( ) )# **params ) )\n str_model_id = 'gpuLogReg'\n \n elif model_cfg[0] == 'gpuCatBClasdefault':\n model = MultiOutputClassifier(estimator= CatBoostClassifier(task_type = 'GPU', verbose = 0 ))# **params ) )\n str_model_id = model_cfg[0] \n\n elif model_cfg[0] == 'gpuCatBdefault': \n model = MultiOutputRegressor(estimator= CatBoostRegressor(task_type = 'GPU', verbose = 0 ))# **params ) )\n str_model_id = model_cfg[0] \n\n elif model_cfg[0] == 'CatBdefault': \n model = MultiOutputRegressor(estimator= CatBoostRegressor(verbose = 0 ))# **params ) )\n str_model_id = model_cfg[0] \n \n elif model_cfg[0] == 'LGBdefault': \n model = MultiOutputRegressor(estimator=lgbm.LGBMRegressor())# **params ) )\n str_model_id = 'LGBdefault'\n # Small mode 46 min on 100 targets \n elif model_cfg[0] == 'skMLP':\n str_model_id = 'skMLP'\n max_iter = model_cfg[1].get('max_iter',500); str_model_id += '_MI'+str(max_iter) \n random_state = model_cfg[1].get('random_state',np.random.randint(0,100)); str_model_id += '_RS'+str(random_state)\n hidden_layer_sizes = model_cfg[1].get('Layers', (100,) ); str_model_id += '_HL'+str(hidden_layer_sizes)\n alpha = model_cfg[1].get('alpha', 1e-4 ); str_model_id += '_alpha'+str(alpha)\n learning_rate_init = model_cfg[1].get('LR', 0.001 ); str_model_id += '_LR'+str(learning_rate_init)\n \n model = MLPRegressor( max_iter=max_iter , random_state=random_state, hidden_layer_sizes = hidden_layer_sizes, alpha= alpha, learning_rate_init = learning_rate_init )\n \n elif model_cfg[0] == 'skMLP2':\n # From: https://www.kaggle.com/code/tttzof351/mmscel-crossvalidation-schemes-193f49\n model = MLPRegressor(max_iter=500, activation='logistic', early_stopping=True,\n solver='adam', alpha=1e-4, random_state= np.random.randint(0,10_000), # 42 \n hidden_layer_sizes=(300, 300))\n str_model_id = 'skMLP2' \n\n elif model_cfg[0] == 'skMLP3':# \n # From: https://www.kaggle.com/code/visualcomments/mmscel-crossvalidation-schemes/notebook#Concatenating-additional-features\n model = MLPRegressor(max_iter=500, activation='logistic', early_stopping=True,\n solver='adam', alpha=1e-5, random_state= 42 , # np.random.randint(0,10_000), # 42, \n hidden_layer_sizes=(300, 200))\n str_model_id = 'skMLP3' \n \n elif model_cfg[0] == 'KMLP3' : # 4Layers concat\n\n str_model_id = 'KMLP3'\n nfeats = X.shape[1]# model_cfg[1]['input_dim']\n nlabels = Y.shape[1]# model_cfg[1]['output_dim']\n \n input_layer = Input(shape=(nfeats,))\n layers_sizes = model_cfg[1].get( 'Layers',[2000,2000,2000,2000] )\n list_reg_l2 = model_cfg[1].get( 'reg_l2', [] )\n list_activations = model_cfg[1].get( 'activation', [] )\n cl = 0\n \n # Define your regular feedforward network\n \n if 1: # Layer 1 \n prm = {}\n if len(layers_sizes) > cl : LS = layers_sizes[cl];\n else: LS = 2000;\n str_model_id += '_L'+str( LS )\n if len(list_reg_l2) > cl : \n prm['kernel_regularizer'] = keras.regularizers.l2( list_reg_l2[cl] )\n str_model_id += '_L2 '+'%.1e'%list_reg_l2[cl] \n if len(list_activations) > cl : \n prm['activation'] = list_activations[cl]\n str_model_id += '_'+str( prm['activation'] )\n else: prm['activation'] = 'relu'\n hidden_layer1 = Dense(LS, **prm)(input_layer)\n cl+=1\n \n BN1 = BatchNormalization()(hidden_layer1)\n dropout1 = Dropout(0.3)(BN1) \n if 1: # Layer 2\n prm = {}\n if len(layers_sizes) > cl : LS = layers_sizes[cl];\n else: LS = 2000;\n str_model_id += '_L'+str( LS )\n if len(list_reg_l2) > cl : \n prm['kernel_regularizer'] = keras.regularizers.l2( list_reg_l2[cl] )\n str_model_id += '_L2 '+'%.1e'%list_reg_l2[cl] \n if len(list_activations) > cl : \n prm['activation'] = list_activations[cl]\n str_model_id += '_'+str( prm['activation'] )\n else: prm['activation'] = 'relu'\n# hidden_layer2 = Dense(LS, **prm)(hidden_layer1)\n hidden_layer2 = Dense(LS, **prm)(dropout1)\n cl+=1\n \n dropout2 = Dropout(0.0)(hidden_layer2) \n \n if 1: # Layer 3\n prm = {}\n if len(layers_sizes) > cl : LS = layers_sizes[cl];\n else: LS = 2000;\n str_model_id += '_L'+str( LS )\n if len(list_reg_l2) > cl : \n prm['kernel_regularizer'] = keras.regularizers.l2( list_reg_l2[cl] )\n str_model_id += '_L2 '+'%.1e'%list_reg_l2[cl] \n if len(list_activations) > cl : \n prm['activation'] = list_activations[cl]\n str_model_id += '_'+str( prm['activation'] )\n else: prm['activation'] = 'relu'\n# hidden_layer3 = Dense(LS, **prm)(hidden_layer2)\n hidden_layer3 = Dense(LS, **prm)(dropout2)\n cl+=1\n\n if 1: # Layer 4\n prm = {}\n if len(layers_sizes) > cl : LS = layers_sizes[cl];\n else: LS = 2000;\n str_model_id += '_L'+str( LS )\n if len(list_reg_l2) > cl : \n prm['kernel_regularizer'] = keras.regularizers.l2( list_reg_l2[cl] )\n str_model_id += '_L2 '+'%.1e'%list_reg_l2[cl] \n if len(list_activations) > cl : \n prm['activation'] = list_activations[cl]\n str_model_id += '_'+str( prm['activation'] )\n else: prm['activation'] = 'relu'\n hidden_layer4 = Dense(LS, **prm)(hidden_layer3)\n cl+=1\n \n # Create the skip connection by concatenating input_layer and hidden_layer2\n concatenated_layers = Concatenate()([hidden_layer1, hidden_layer2, hidden_layer3, hidden_layer4])\n\n # # Add another hidden layer after the concatenated skip connection\n # hidden_layer3 = Dense(32, activation='relu')(concatenated_layers)\n\n if 1: # Create the output layer\n prm = {}\n if len(list_reg_l2) > cl : \n prm['kernel_regularizer'] = keras.regularizers.l2( list_reg_l2[cl] )\n str_model_id += '_L2 '+'%.1e'%list_reg_l2[cl] \n if len(list_activations) > cl : \n str_model_id += '_'+str( prm['activation'] )\n else: prm['activation'] = 'sigmoid'\n output_layer = Dense(nlabels, **prm)(concatenated_layers)\n cl+=1\n \n # Define the model with the input and output layers\n model = Model(inputs=input_layer, outputs=output_layer)\n\n # Compile the model with appropriate optimizer, loss, and metrics\n str_optimizer = model_cfg[1].get('optimizer', 'adam')\n learning_rate = model_cfg[1].get('LR', 0.001 ) \n if str_optimizer == 'adam':\n optimizer = Adam(lr=learning_rate)\n elif str_optimizer == 'RMSprop':\n optimizer = RMSprop(lr=learning_rate)\n elif str_optimizer == 'SGD':\n optimizer = SGD(lr=learning_rate)\n if str_optimizer != 'adam': str_model_id += '_'+str_optimizer\n if learning_rate != 0.001: str_model_id += '_LR'+'%.2e'%( learning_rate )\n \n model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=[AUC()]) \n\n # verbose = 100000\n if 1: # verbose >= 10:\n print(str_model_id )\n display(plot_model(model, show_layer_names=False, show_shapes=True, dpi=72))\n\n # Print the model summary\n model.summary()\n\n \n elif model_cfg[0] == 'KMLP2' :\n\n str_model_id = 'KMLP2'\n nfeats = X.shape[1]# model_cfg[1]['input_dim']\n nlabels = Y.shape[1]# model_cfg[1]['output_dim']\n \n input_layer = Input(shape=(nfeats,))\n layers_sizes = model_cfg[1].get( 'Layers',[2000,2000,2000] )\n list_reg_l2 = model_cfg[1].get( 'reg_l2', [] )\n list_activations = model_cfg[1].get( 'activation', [] )\n cl = 0\n \n # Define your regular feedforward network\n \n if 1: # Layer 1 \n prm = {}\n if len(layers_sizes) > cl : LS = layers_sizes[cl];\n else: LS = 2000;\n str_model_id += '_L'+str( LS )\n if len(list_reg_l2) > cl : \n prm['kernel_regularizer'] = keras.regularizers.l2( list_reg_l2[cl] )\n str_model_id += '_L2 '+'%.1e'%list_reg_l2[cl] \n if len(list_activations) > cl : \n prm['activation'] = list_activations[cl]\n str_model_id += '_'+str( prm['activation'] )\n else: prm['activation'] = 'relu'\n hidden_layer1 = Dense(LS, **prm)(input_layer)\n cl+=1\n\n dropout1 = Dropout(0.3)(hidden_layer1) \n if 1: # Layer 2\n prm = {}\n if len(layers_sizes) > cl : LS = layers_sizes[cl];\n else: LS = 2000;\n str_model_id += '_L'+str( LS )\n if len(list_reg_l2) > cl : \n prm['kernel_regularizer'] = keras.regularizers.l2( list_reg_l2[cl] )\n str_model_id += '_L2 '+'%.1e'%list_reg_l2[cl] \n if len(list_activations) > cl : \n prm['activation'] = list_activations[cl]\n str_model_id += '_'+str( prm['activation'] )\n else: prm['activation'] = 'relu'\n #hidden_layer2 = Dense(LS, **prm)(hidden_layer1)\n hidden_layer2 = Dense(LS, **prm)(dropout1)\n cl+=1\n\n # Create the skip connection by concatenating input_layer and hidden_layer2\n concatenated_layers = Concatenate()([hidden_layer1, hidden_layer2])\n\n # # Add another hidden layer after the concatenated skip connection\n # hidden_layer3 = Dense(32, activation='relu')(concatenated_layers)\n\n if 1: # Create the output layer\n prm = {}\n if len(list_reg_l2) > cl : \n prm['kernel_regularizer'] = keras.regularizers.l2( list_reg_l2[cl] )\n str_model_id += '_L2 '+'%.1e'%list_reg_l2[cl] \n if len(list_activations) > cl : \n str_model_id += '_'+str( prm['activation'] )\n else: prm['activation'] = 'sigmoid'\n output_layer = Dense(nlabels, **prm)(concatenated_layers)\n cl+=1\n \n # Define the model with the input and output layers\n model = Model(inputs=input_layer, outputs=output_layer)\n\n \n # Compile the model with appropriate optimizer, loss, and metrics\n str_optimizer = model_cfg[1].get('optimizer', 'adam')\n learning_rate = model_cfg[1].get('LR', 0.001 ) \n if str_optimizer == 'adam':\n optimizer = Adam(lr=learning_rate)\n elif str_optimizer == 'RMSprop':\n optimizer = RMSprop(lr=learning_rate)\n elif str_optimizer == 'SGD':\n optimizer = SGD(lr=learning_rate)\n if str_optimizer != 'adam': str_model_id += '_'+str_optimizer\n if learning_rate != 0.001: str_model_id += '_LR'+'%.2e'%( learning_rate )\n \n model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=[AUC()]) \n\n # verbose = 100000\n if 1: # verbose >= 10:\n print(str_model_id )\n display(plot_model(model, show_layer_names=False, show_shapes=True, dpi=72))\n\n # Print the model summary\n model.summary()\n\n\n elif model_cfg[0] == 'KMLP' :\n \n model = Sequential(); str_model_id = 'KMLP'\n layers_sizes = model_cfg[1]['Layers']\n if 'Dropouts' in model_cfg[1].keys():\n list_droupouts = model_cfg[1]['Dropouts']\n else:\n list_droupouts = []\n if 'BatchNormalizations' in model_cfg[1].keys():\n list_batchnormalization = model_cfg[1]['BatchNormalizations']\n else:\n list_batchnormalization = []\n \n nfeats = X.shape[1]# model_cfg[1]['input_dim']\n nlabels = Y.shape[1]# model_cfg[1]['output_dim']\n \n ######### First Layer #################################################################################\n i_layer = 0 \n layer_dim = layers_sizes[i_layer]\n model.add(Dense(layer_dim, activation='relu', input_dim=nfeats, kernel_regularizer = keras.regularizers.l2( 0 ) ) ) ; str_model_id += '_L1_'+str(layer_dim) \n if len( list_batchnormalization ) > i_layer:\n if list_batchnormalization[i_layer]:\n model.add(BatchNormalization())\n str_model_id += '_BN'\n if (len( list_droupouts ) > i_layer) and ( list_droupouts[i_layer] is not None ):\n droupout_rate = list_droupouts[i_layer]\n model.add(Dropout(droupout_rate))\n str_model_id += '_DR'+str( np.round(droupout_rate ,2) ) \n \n ######### Middle layers ##########################################################################################\n for i_layer in range(1, len( layers_sizes ) ) : \n layer_dim = layers_sizes[i_layer]\n if layer_dim is None: break \n model.add(Dense(layer_dim, activation='relu' , kernel_regularizer = keras.regularizers.l2( 0 ) )); str_model_id += '_L'+str(i_layer+1)+'_'+str(layer_dim) \n# model.add(LayerNormalization())\n if len( list_batchnormalization ) > i_layer:\n if list_batchnormalization[i_layer]:\n model.add(BatchNormalization())\n str_model_id += '_BN'\n if (len( list_droupouts ) > i_layer) and ( list_droupouts[i_layer] is not None ):\n droupout_rate = list_droupouts[i_layer]\n model.add(Dropout(droupout_rate))\n str_model_id += '_DR'+str( np.round(droupout_rate ,2) ) \n \n ######### Last layer ########################################################################################\n model.add(Dense(nlabels, activation='sigmoid' , kernel_regularizer = keras.regularizers.l2( 0 ) ))\n# model.add(Dense(nlabels, activation='tanh'))\n #model.add(Dense(nlabels, activation='relu'))\n \n \n model.compile(loss='binary_crossentropy',\n optimizer='adam',\n metrics=[AUC()]) \n \n if verbose >= 10:\n print( str_model_id, model )\n \n \n return model, str_model_id","metadata":{"papermill":{"duration":0.036114,"end_time":"2023-07-15T21:45:08.074328","exception":false,"start_time":"2023-07-15T21:45:08.038214","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-20T20:35:04.720092Z","iopub.execute_input":"2023-07-20T20:35:04.720564Z","iopub.status.idle":"2023-07-20T20:35:04.781893Z","shell.execute_reply.started":"2023-07-20T20:35:04.720513Z","shell.execute_reply":"2023-07-20T20:35:04.780429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model_fit(model , X_loc, Y_loc, config_model_features_postprocessing_etc ):\n \n model_cfg = config_model_features_postprocessing_etc['Model']\n \n if verbose >= 100:\n print( model_cfg )\n \n dict_prm_for_fit = {t : model_cfg[1][t] for t in ['epochs' , 'batch_size' , 'verbose'] if t in model_cfg[1].keys() }\n \n # if ('epochs' in model_cfg[1].keys() ) and ( 'batch_size' in model_cfg[1].keys() ) and ('verbose' in model_cfg[1].keys() ) :\n \n if len( dict_prm_for_fit ) == 0:\n model.fit( X_loc, Y_loc )\n else:\n model.fit( X_loc, Y_loc , **dict_prm_for_fit )\n ","metadata":{"papermill":{"duration":0.027384,"end_time":"2023-07-15T21:45:08.117994","exception":false,"start_time":"2023-07-15T21:45:08.09061","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-20T20:35:04.783901Z","iopub.execute_input":"2023-07-20T20:35:04.784252Z","iopub.status.idle":"2023-07-20T20:35:04.793006Z","shell.execute_reply.started":"2023-07-20T20:35:04.784215Z","shell.execute_reply":"2023-07-20T20:35:04.791904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features( config_model_features_postprocessing_etc , verbose = 100_000 ): \n if 'Features' in config_model_features_postprocessing_etc.keys():\n dict_features_cfg = config_model_features_postprocessing_etc['Features']\n else:\n print('return default' )\n return X, 'Default'\n \n if verbose >=10_000:\n print(dict_features_cfg )\n \n str_data_id = 'todo'\n if 'list_features_ids' in dict_features_cfg.keys():\n list_features = dict_features_cfg['list_features_ids']\n if verbose >= 100:\n print('list_features', list_features )\n if list_features == ['T5_43k']:\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n X = np.load(fn)\n print('X.shape', X.shape, 'loaded')\n elif set(list_features) == set(['T5_43k', 'esm2Fairushchin_43k']):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_esm2Fairushchin_float32_orderedAsInput_cut43k.npy'\n X = np.concatenate( [np.load(fn), np.load(fn2)], axis =1)\n elif set(list_features) == set(['T5_43k', 'esm2Fairushchin_43k','taxons31_onehot_43k']):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_esm2Fairushchin_float32_orderedAsInput_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32)], axis =1)\n elif set(list_features) == set(['T5_43k', 'esm2Fairushchin_43k','taxons31_onehot_43k', 'F06_genes' ]):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_esm2Fairushchin_float32_orderedAsInput_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n fn4 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features06_genes_names_related_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),\n pd.read_csv(fn4,index_col = 0).values.astype(np.float32) ], axis =1)\n elif set(list_features) == set(['T5_43k', 'esm2Fairushchin_43k','taxons31_onehot_43k', 'F07_DesriptKw1H' ]):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_esm2Fairushchin_float32_orderedAsInput_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n fn4 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features07_keywords_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),\n pd.read_csv(fn4,index_col = 0).values.astype(np.float32) ], axis =1)\n\n elif set(list_features) == set(['T5_43k', 'esm2S2560_43k', 'taxons31_onehot_43k' ]):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t36B3S2560_train_embeds_float32_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),\n ], axis =1)\n elif set(list_features) == set(['T5_43k', 'esm2S1280_43k', 'taxons31_onehot_43k' ]):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t33M650S1280_train_embeds_float32_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),\n ], axis =1)\n\n elif set(list_features) == set(['T5_43k', 'esm2S480_43k', 'taxons31_onehot_43k' ]):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t12M35S480_train_embeds_float32_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),\n ], axis =1)\n elif set(list_features) == set(['T5_43k', 'protb_43k', 'taxons31_onehot_43k' ]):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_protb_train_float32_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),\n ], axis =1)\n elif set(list_features) == set([ 'T5_43k', 'esm2S480_43k' , 'protb_43k' , 'taxons31_onehot_43k']):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t12M35S480_train_embeds_float32_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n fn4 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_protb_train_float32_cut43k.npy'\n\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),\n np.load(fn4), ], axis =1)\n \n elif set(list_features) == set([ 'T5_43k', 'esm2S2560cls_43k' , 'taxons31_onehot_43k']):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t36B3S2560cls_train_float32_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),], axis =1)\n #dict_data_cfg3 = {'list_features_ids': ['T5_43k', 'esm2S2560cls_43k' , 'taxons31_onehot_43k' ] } \n # list_features = ['T5_43k', 'esm2S2560cls_43k' , 'protb_43k' , 'taxons31_onehot_43k' ] # , 'F06_genes' # ['T5_43k']# \n # /kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t36B3S2560cls_train_float32_cut43k.npy\n \n elif set(list_features) == set([ 'T5_43k', 'E640_43k' , 'taxons31_onehot_43k']):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t30M150S640_train_embeds_float32_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),], axis =1)\n # list_features = ['T5_43k', 'E640_43k' , 'taxons31_onehot_43k' ] # , 'F06_genes' # ['T5_43k']# \n # /kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t30M150S640_train_embeds_float32_cut43k.npy\n\n elif set(list_features) == set([ 'T5_43k', 'E2560_43k' , 'taxons31_onehot_43k']):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t36B3S2560_train_embeds_float32_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),], axis =1)\n\n # dict_data_cfg5 = {'list_features_ids': [ 'T5_43k','E2560_43k' , 'taxons31_onehot_43k'] }\n # /kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t36B3S2560_train_embeds_float32_cut43k.npy\n \n elif set(list_features) == set([ 'T5_43k','E320_43k' , 'E480_43k' , 'taxons31_onehot_43k']):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t6M8S320_train_embeds_float32_cut43k.npy'\n fn3 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t12M35S480_train_embeds_float32_cut43k.npy'\n fn4 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), np.load(fn3),\n pd.read_csv(fn4,index_col = 0).values.astype(np.float32), ], axis =1)\n elif set(list_features) == set([ 'T5_43k', 'E2560mean_43k' , 'taxons31_onehot_43k']):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t36B3S2560mean_train_float32_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),], axis =1)\n\n elif set(list_features) == set([ 'T5_43k', 'E1280_43k' , 'taxons31_onehot_43k']):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t33M650S1280_train_embeds_float32_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),], axis =1)\n elif set(list_features) == set([ 'T5_43k', 'protb_43k' , 'taxons31_onehot_43k']):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_protb_train_float32_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),], axis =1)\n elif set(list_features) == set([ 'T5_43k', 'E5120_43k' , 'taxons31_onehot_43k']):\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_train_T5_float32_cut43k.npy'\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t48B15S5120_train_float32_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n X = np.concatenate( [np.load(fn), np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),], axis =1)\n elif set(list_features) == set([ 'E5120_43k']):\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t48B15S5120_train_float32_cut43k.npy'\n X = np.load(fn2)\n elif set(list_features) == set([ 'E5120_43k' , 'taxons31_onehot_43k']):\n fn2 = '/kaggle/input/protein-embeddings-4-cafa5-selected/Embeddings_43k/emb_esm2t48B15S5120_train_float32_cut43k.npy'\n fn3 = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/train_features05_taxons31_onehot_cut43k.csv'\n X = np.concatenate( [ np.load(fn2), pd.read_csv(fn3,index_col = 0).values.astype(np.float32),], axis =1)\n \n \n else:\n print('Something wrong - parameters not found !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')\n else:\n print('Something wrong - no key to process - parameters not found !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')\n \n\n\n \n X = X[:n_samples_to_consider,:]\n\n if verbose >= 100:\n print('X.shape', X.shape, 'effective') \n print('type(X[0,0])' , type(X[0,0]) )\n print(X[:3,:2] , X.sum() ) \n \n \n return X , str_data_id","metadata":{"papermill":{"duration":0.059539,"end_time":"2023-07-15T21:45:08.193702","exception":false,"start_time":"2023-07-15T21:45:08.134163","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-20T20:35:04.794662Z","iopub.execute_input":"2023-07-20T20:35:04.795301Z","iopub.status.idle":"2023-07-20T20:35:04.857702Z","shell.execute_reply.started":"2023-07-20T20:35:04.795263Z","shell.execute_reply":"2023-07-20T20:35:04.856314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if (mode_submit is not None) and ( mode_submit != False ): # None\n# X_submit, str_submit_data_id = get_features_submit( config_model_features_postprocessing_etc ) # , verbose = 100_000 )\ndef get_features_submit( config_model_features_postprocessing_etc , verbose = 100_000 ):\n \n if 'Features' in config_model_features_postprocessing_etc.keys():\n dict_features_cfg = config_model_features_postprocessing_etc['Features']\n else:\n print('return default' )\n return X_submit, 'Default'\n \n if verbose >=10_000:\n print(dict_features_cfg )\n \n str_data_id = 'todo'\n if 'list_features_ids' in dict_features_cfg.keys():\n list_features = dict_features_cfg['list_features_ids']\n print('list_features', list_features )\n\n X_submit = np.zeros( (141865,0), dtype = np.float32 )\n if 'T5' in str(list_features) :\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_submit_T5_float32.npy'\n if verbose >=100: print(fn)\n X_submit = np.concatenate( [X_submit, np.load(fn)], axis =1)\n if 'esm2Fairushchin' in str(list_features) :\n fn = '/kaggle/input/cafa5-data-selected/embeds_43k_Y1850_etc/embeds_submit_esm2Fairushchin_float32_orderedAsInput.npy'\n if verbose >=100: print(fn)\n X_submit = np.concatenate( [X_submit, np.load(fn)], axis =1)\n if 'taxons31_onehot' in str(list_features): \n fn = '/kaggle/input/cafa5-data-selected/features/test_features05_taxons31_onehot.csv'\n if verbose >=100: print(fn)\n X_submit = np.concatenate( [X_submit, pd.read_csv(fn,index_col = 0).values.astype(np.float32)], axis =1)\n if 'F06_genes' in str(list_features): \n fn = '/kaggle/input/cafa5-data-selected/features/test_features06_genes_names_related_onehot.csv'\n if verbose >=100:print(fn)\n X_submit = np.concatenate( [X_submit, pd.read_csv(fn,index_col = 0).values.astype(np.float32)], axis =1)\n if 'F07_DesriptKw1H' in str(list_features): \n fn = '/kaggle/input/cafa5-data-selected/features/test_features07_keywords_onehot.csv'\n if verbose >=100: print(fn)\n X_submit = np.concatenate( [X_submit, pd.read_csv(fn,index_col = 0).values.astype(np.float32)], axis =1)\n\n if 'esm2S480' in str(list_features): \n fn = '/kaggle/input/protein-embeddings-2-cafa5/Embeddings_test/emb_esm2t12M35S480_test_float32.npy'\n if verbose >=100: print(fn)\n X_submit = np.concatenate( [X_submit, np.load(fn)], axis =1)\n if 'E480' in str(list_features): \n fn = '/kaggle/input/protein-embeddings-2-cafa5/Embeddings_test/emb_esm2t12M35S480_test_float32.npy'\n if verbose >=100: print(fn)\n X_submit = np.concatenate( [X_submit, np.load(fn)], axis =1)\n \n if 'esm2S2560cls' in str(list_features): \n fn = '/kaggle/input/protein-embeddings-2-cafa5/Embeddings_test/emb_esm2t36B3S2560cls_test_float32.npy'\n if verbose >=100: print(fn)\n X_submit = np.concatenate( [X_submit, np.load(fn)], axis =1)\n if 'E640' in str(list_features): \n fn = '/kaggle/input/protein-embeddings-2-cafa5/Embeddings_test/emb_esm2t30M150S640_test_float32.npy'\n if verbose >=100: print(fn)\n X_submit = np.concatenate( [X_submit, np.load(fn)], axis =1)\n if 'E2560_' in str(list_features): \n fn = '/kaggle/input/protein-embeddings-2-cafa5/Embeddings_test/emb_esm2t36B3S2560_test_float32.npy'\n if verbose >=100: print(fn)\n X_submit = np.concatenate( [X_submit, np.load(fn)], axis =1)\n if 'protb' in str(list_features): \n fn = '/kaggle/input/protein-embeddings-2-cafa5/Embeddings_test/emb_protb_test_float32.npy'\n if verbose >=100: print(fn)\n X_submit = np.concatenate( [X_submit, np.load(fn)], axis =1)\n if 'E2560sum' in str(list_features): \n fn = '/kaggle/input/protein-embeddings-2-cafa5/Embeddings_test/emb_esm2t36B3S2560sum_test_float32.npy'\n if verbose >=100: print(fn)\n X_submit = np.concatenate( [X_submit, np.load(fn)], axis =1)\n if 'E2560mean' in str(list_features): \n fn = '/kaggle/input/protein-embeddings-2-cafa5/Embeddings_test/emb_esm2t36B3S2560mean_test_float32.npy'\n if verbose >=100: print(fn)\n X_submit = np.concatenate( [X_submit, np.load(fn)], axis =1)\n if 'sbert' in str(list_features): \n fn = '/kaggle/input/protein-embeddings-2-cafa5/Embeddings_test/emb_sbert_test_float32.npy'\n if verbose >=100: print(fn)\n X_submit = np.concatenate( [X_submit, np.load(fn)], axis =1)\n if 'E1280_' in str(list_features): \n fn = '/kaggle/input/protein-embeddings-2-cafa5/Embeddings_test/emb_esm2t33M650S1280_test_float32.npy'\n if verbose >=100: print(fn)\n X_submit = np.concatenate( [X_submit, np.load(fn)], axis =1)\n if 'sgt' in str(list_features): \n fn = '/kaggle/input/protein-embeddings-2-cafa5/Embeddings_test/emb_sgt_test_float32.npy'\n if verbose >=100: print(fn)\n X_submit = np.concatenate( [X_submit, np.load(fn)], axis =1)\n if 'E320' in str(list_features): \n fn = '/kaggle/input/protein-embeddings-2-cafa5/Embeddings_test/emb_esm2t6M8S320_test_float32.npy'\n if verbose >=100: print(fn)\n X_submit = np.concatenate( [X_submit, np.load(fn)], axis =1)\n if 'E5120_' in str(list_features): \n fn = '/kaggle/input/protein-embeddings-3-cafa5/Embeddings_test/emb_esm2t48B15S5120_test_float32.npy'\n fn = '/kaggle/input/protein-embeddings-3-cafa5/Embeddings_test/emb_esm2t48B15S5120_test_embeds_float32.npy'\n if verbose >=100: print(fn)\n X_submit = np.concatenate( [X_submit, np.load(fn)], axis =1)\n \n \n if X.shape[1] == 0:\n print('X_submit : Something wrong - parameters not found !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')\n else:\n print('X_submit : Something wrong - no key to process - parameters not found !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')\n \n if verbose >= 100:\n print('X_submit:')\n print( X_submit.shape, type(X[0,0]) )\n print( X_submit[:3,:2] )\n\n return X_submit, str_data_id","metadata":{"execution":{"iopub.status.busy":"2023-07-20T20:35:04.860809Z","iopub.execute_input":"2023-07-20T20:35:04.861463Z","iopub.status.idle":"2023-07-20T20:35:04.906565Z","shell.execute_reply.started":"2023-07-20T20:35:04.8614Z","shell.execute_reply":"2023-07-20T20:35:04.905045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling","metadata":{"papermill":{"duration":0.016335,"end_time":"2023-07-15T21:45:08.226418","exception":false,"start_time":"2023-07-15T21:45:08.210083","status":"completed"},"tags":[]}},{"cell_type":"code","source":"pd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\n# pd.set_option('max_colwidth', -1)","metadata":{"execution":{"iopub.status.busy":"2023-07-20T20:35:04.914298Z","iopub.execute_input":"2023-07-20T20:35:04.91485Z","iopub.status.idle":"2023-07-20T20:35:04.92505Z","shell.execute_reply.started":"2023-07-20T20:35:04.914785Z","shell.execute_reply":"2023-07-20T20:35:04.922882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nimport gc\nimport time \n\n##################################################################\n## Params: \n##################################################################\n\nverbose = 1000\n\n##################################################################\n## Results: \n##################################################################\ndf_foldwise_stat = pd.DataFrame()\n\n\ncnt_blend_submit = 0 ; \nif (mode_submit is not None) and ( mode_submit != False ): # None\n Y_submit = np.zeros( (141865, Y.shape[1] ) , dtype = np.float16 ) \n\n# if verbose >= 100:\n# print(str_model_id, str_cv_info_default, 'Y.shape:', Y.shape )\n\n# cfg0 = {'Model': ('Ridge',10)}\n# cfg1 = {'Model': ('KMLP', {'Layers': [100], 'input_dim': X.shape[1] , 'output_dim': Y.shape[1] } ) }\nlist_models_ids = []\n\nY_pred_oof = np.zeros(Y.shape , dtype = np.float16 ) ; cnt_blend_oof = -1;\n\nt0modeling = time.time(); i_modeling = 0;\n##################################################################\n## Loop over models\n##################################################################\n\nprint(); print(); print('Modeling starts.');print(); print(); \nfor i_cfg, config_model_features_postprocessing_etc in enumerate( list_main_configs_models_etc): # [cfg0, cfg1 ]:\n print( config_model_features_postprocessing_etc )\n \n if 'Features' in config_model_features_postprocessing_etc.keys():\n X , str_data_id = get_features( config_model_features_postprocessing_etc )\n \n if (mode_submit is not None) and ( mode_submit != False ): # None\n X_submit, str_submit_data_id = get_features_submit( config_model_features_postprocessing_etc ) # , verbose = 100_000 )\n if X_submit.shape[1] != X.shape[1]:\n print(); print(); print('ERROR !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!');\n print('submit shape different from X shape'); print(); print(); \n raise Exception(\"ERROR - submit shape different from X shape\")\n \n cnt_blend_oof += 1; \n Y_pred_oof_one_model = np.zeros( Y.shape , dtype = np.float16 )\n\n str_full_id = 'C'+str(i_cfg)\n if 'n_folds' in config_model_features_postprocessing_etc.keys():\n n_folds = config_model_features_postprocessing_etc['n_folds']\n str_full_id += ' F'+str(n_folds)\n str_cv_info = 'cv_folds_'+str(n_folds)\n vec_folds_labels = df_folds['cv_folds_'+str(n_folds) ] # 0,1,2,3... - ids of folds for each sample \n else:\n vec_folds_labels = df_folds[str_cv_info_default] # 0,1,2,3... - ids of folds for each sample \n str_cv_info = str_cv_info_default\n\n fit_time = 0; predict_time = 0; scoring_time = 0; t0model_start = time.time();\n \n ##################################################################\n ## Loop over folds \n ##################################################################\n for i_fold,uv in enumerate(np.sort(vec_folds_labels.unique())[:n_folds_to_process]):\n\n ########## Get folds #########################################\n m = vec_folds_labels == uv\n IX_train = np.where( ~m)[0]\n IX_test = np.where( m)[0]\n if verbose >= 100:\n print('Folds info:',i_fold,uv, m.sum() )\n\n ########## Get model #########################################\n model, str_model_id = get_model( config_model_features_postprocessing_etc , verbose = 10 )\n if i_fold == 0: \n str_full_id = str_model_id + ' ' +str_full_id\n list_models_ids.append( str_model_id )\n \n ########## Fit model #########################################\n t0 = time.time()\n #model.fit(X[IX_train,:], Y[IX_train,:] )\n # model.fit(X[IX_train,:],Y[IX_train,:], epochs=epochs, batch_size=batch_size) # , verbose = 0\n model_fit(model , X[IX_train,:],Y[IX_train,:], config_model_features_postprocessing_etc )\n fit_time_fold = ( time.time() - t0 ); fit_time += fit_time_fold \n if verbose >= 100:\n print('%.1f secs passed on fit'%(fit_time_fold)) \n\n\n ########## Predict on local test, OOF #########################################\n flag_predict_local_test = True \n if flag_predict_local_test: \n IX_loc = IX_test\n t0 = time.time()\n Y_pred = model.predict(X[IX_loc,:])\n time_predict_on_test_fold = time.time() - t0\n Y_pred_oof_one_model[IX_loc,:] = Y_pred \n if verbose >= 100:\n print('%.1f secs passed on predict local test'%(time_predict_on_test_fold)) \n\n if blend_mode == 'mean':\n Y_pred_oof[IX_loc,:] = (Y_pred_oof[IX_loc,:] * cnt_blend_oof + Y_pred )/ (cnt_blend_oof + 1); \n elif blend_mode == 'max':\n Y_pred_oof[IX_loc,:] = np.maximum(Y_pred_oof[IX_loc,:] , Y_pred )\n \n \n if flag_scoring_each_model:\n update_scores(df_foldwise_stat, str_full_id, fold_id = uv, Y = Y[IX_loc,:] , Y_pred = Y_pred, \n IX = IX_loc, cutoff_threshold_low = cutoff_threshold_low , fit_time = fit_time_fold,\n predict_time = time_predict_on_test_fold, blend_mode = blend_mode, str_cv_info = str_cv_info )\n if (flag_scoring_each_blend): # and (cnt_blend_oof > 0):\n if not ( (cnt_blend_oof == 0) and flag_scoring_each_model ): # blend1 is the same as the first model - no need to duplicate \n update_scores(df_foldwise_stat, 'blend'+str(cnt_blend_oof+1), fold_id = uv, Y = Y[IX_loc,:] , Y_pred = Y_pred_oof[IX_loc,:], \n IX = IX_loc, cutoff_threshold_low = cutoff_threshold_low , fit_time = fit_time_fold,\n predict_time = time_predict_on_test_fold, blend_mode = blend_mode, str_cv_info = str_cv_info )\n\n display(df_foldwise_stat)\n \n if (mode_submit is not None) and ( mode_submit != False ): # None\n \n if blend_mode == 'mean':\n Y_submit = (Y_submit * cnt_blend_submit + model.predict(X_submit) )/ (cnt_blend_submit + 1); \n elif blend_mode == 'max':\n Y_submit = np.maximum(Y_submit , model.predict(X_submit) )\n cnt_blend_submit += 1 \n \n if verbose >= 1_000:\n try:\n print( model.summary() )\n except:\n pass\n\n del model \n gc.collect()\n \nprint('Modeling finished. %.1f secs passed.'%( time.time() - t0modeling ) )\n \n \n########## Final scoring #########################################\n# If scoring is done on each step - that it is not necessary, but scoring is quite long , sometimes we need only the final scoring \nif flag_scoring_final:\n t0 = time.time()\n str_info = 'FinalBlend'+str(cnt_blend_oof+1)\n # for i_fold,uv in enumerate(np.sort(vec_folds_labels.unique())):\n for i_fold,uv in enumerate(np.sort(vec_folds_labels.unique())[:n_folds_to_process]): \n\n ########## Get folds #########################################\n m = vec_folds_labels == uv\n IX_train = np.where( ~m)[0]\n IX_test = np.where( m)[0]\n IX_loc = IX_test\n \n ########## Scoring #########################################\n update_scores(df_foldwise_stat, str_info, fold_id = uv, Y = Y[IX_loc,:] , Y_pred = Y_pred_oof[IX_loc,:], \n IX = IX_loc, cutoff_threshold_low = cutoff_threshold_low , fit_time = fit_time_fold,\n predict_time = time_predict_on_test_fold, blend_mode = blend_mode, str_cv_info = str_cv_info )\n df_foldwise_stat.loc[len(df_foldwise_stat), 'All Models'] = str( list_models_ids )\n print('Final Blend scoring finished. %.1f secs passed.'%( time.time() - t0 ) )\n\nif 1:\n try:\n t0 = time.time()\n s = roc_auc_score(Y.ravel(), Y_pred_oof.ravel() )\n print('Roc auc scoring finished. %.1f secs passed.'%( time.time() - t0 ) )\n print(s)\n except:\n print('Exception in roc auc computation')\n \n# display(df_foldwise_stat.corr(method = 'spearman').round(2)) \ndisplay(df_foldwise_stat)\nif len(df_foldwise_stat) > 0:\n d_ = df_foldwise_stat.groupby('FullId').mean().sort_values('F1w',ascending = False ).round(6)\n display(d_ )\n d_.to_csv('df_models_stat.csv')\n d2_ = df_foldwise_stat.groupby('FullId').std().sort_values('F1w',ascending = False ).round(6)\n display(d2_ )\n d2_.to_csv('df_models_std_stat.csv')\n\ngc.collect()\n","metadata":{"papermill":{"duration":635.124359,"end_time":"2023-07-15T21:55:43.367486","exception":false,"start_time":"2023-07-15T21:45:08.243127","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-20T20:35:04.927771Z","iopub.execute_input":"2023-07-20T20:35:04.928178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n print( model.summary() )\nexcept:\n pass\n","metadata":{"papermill":{"duration":0.188785,"end_time":"2023-07-15T21:55:43.737167","exception":false,"start_time":"2023-07-15T21:55:43.548382","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Show Results","metadata":{"papermill":{"duration":0.178072,"end_time":"2023-07-15T21:55:44.096436","exception":false,"start_time":"2023-07-15T21:55:43.918364","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if len(df_foldwise_stat) > 0:\n d_ = df_foldwise_stat.groupby('FullId').mean().sort_values('F1w',ascending = False ).round(6)\n display(d_ )\n d_.to_csv('df_models_stat.csv')\n d2_ = df_foldwise_stat.groupby('FullId').std().sort_values('F1w',ascending = False ).round(6)\n display(d2_ )\n d2_.to_csv('df_models_std_stat.csv')\n d_['F1w']","metadata":{"papermill":{"duration":0.236432,"end_time":"2023-07-15T21:55:44.51233","exception":false,"start_time":"2023-07-15T21:55:44.275898","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if len(df_foldwise_stat) > 0:\n m = np.array( [ 'blend' not in t for t in d_.index ] )\n display(d_[m])\n if m.sum() > 0:\n d_[m].to_csv('df_models_no_blend_stat.csv')\n display( d_[m].describe() )\n d_[m].describe().to_csv('df_models_no_blend_describe.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.sum(Y.astype(float))/Y.shape[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if len( df_foldwise_stat ) > 0:\n display(df_foldwise_stat.corr(method = 'spearman').round(2)) \n display(df_foldwise_stat) \n","metadata":{"papermill":{"duration":0.239428,"end_time":"2023-07-15T21:55:44.932024","exception":false,"start_time":"2023-07-15T21:55:44.692596","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 0.9048271262243464 - rocauc oof T5 3 folds\n# 0.8959511674801596 - rocauc oof T5+esm2 3 folds","metadata":{"papermill":{"duration":0.188298,"end_time":"2023-07-15T21:55:45.300272","exception":false,"start_time":"2023-07-15T21:55:45.111974","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ngc.collect()","metadata":{"papermill":{"duration":0.444395,"end_time":"2023-07-15T21:55:45.923022","exception":false,"start_time":"2023-07-15T21:55:45.478627","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission part","metadata":{"papermill":{"duration":0.235463,"end_time":"2023-07-15T21:55:46.337632","exception":false,"start_time":"2023-07-15T21:55:46.102169","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time \n\nif (mode_submit is not None) and ( mode_submit != False ): # None\n\n if 1:\n del X\n del Y\n del Y_pred_oof_one_model\n del Y_pred_oof\n del X_submit\n del trainTerms\n gc.collect()\n \n print( Y_submit.shape)\n Y_submit[:3,:2] \n \n fn = '/kaggle/input/t5embeds/test_ids.npy'\n vec_test_protein_ids = np.load(fn)\n print(len(vec_test_protein_ids), vec_test_protein_ids[:10])\n \n IX = np.where(Y_submit > cutoff_threshold_low )\n print( len(IX[0]), len(IX[0]) /1e6 )\n vec_protein_ids_loc = vec_test_protein_ids\n print(len(labels_to_consider), labels_to_consider[:10] ) \n preds = []\n print('Start creating submission dataframe')\n for i in range(len(IX[0])):\n preds.append(\\\n (vec_protein_ids_loc[IX[0][i]], labels_to_consider[IX[1][i]], np.round( Y_submit[IX[0][i],IX[1][i]],3) )\\\n )\n if (i%1_000_000 == 0): print(i)\n df_finalSubmission = pd.DataFrame(data = preds, columns = ['Protein Id', 'GO Term Id','Prediction'])\n display( df_finalSubmission.head(10) )\n print( 'df_finalSubmission.memory_usage Ms:', df_finalSubmission.memory_usage().sum()/1e6 )\n \n del Y_submit\n del preds \n del IX \n gc.collect()\n\n df_finalSubmission.to_csv(\"submission.tsv\",header=False, index=False, sep=\"\\t\")\n \n gc.collect()\n\n# ## Old way with greater memory usage\n# if 0:\n# if (mode_submit is not None) and ( mode_submit != False ): # None\n\n# if 1:\n# del X\n# del Y\n# del Y_pred_oof_one_model\n# del Y_pred_oof\n# del X_submit\n# del trainTerms\n# gc.collect()\n\n# print( Y_submit.shape)\n# Y_submit[:3,:2] \n\n\n# fn = '/kaggle/input/t5embeds/test_ids.npy'\n# vec_test_protein_ids = np.load(fn)\n# print(vec_test_protein_ids.shape)\n# print( vec_test_protein_ids[:10] )\n\n# l = []\n# for k in list(vec_test_protein_ids):\n# l += [ k] * Y_submit.shape[1]\n# print(len(l), l[:20]) \n# df_finalSubmission['Protein Id'] = l\n\n\n# df_finalSubmission['GO Term Id'] = list(labels_to_consider) * Y_submit.shape[0]\n\n\n# df_finalSubmission['Prediction'] = np.clip( np.round(Y_submit.ravel(),3),0.01,1 ) \n\n\n# print(df_finalSubmission.shape, len(df_finalSubmission)/1e6 )\n# m = df_finalSubmission['Prediction'] > cutoff_threshold_low\n# df_finalSubmission = df_finalSubmission[m]\n# print(df_finalSubmission.shape)\n\n\n# display( df_finalSubmission.head(10) )\n# df_finalSubmission.to_csv(\"submission.tsv\",header=False, index=False, sep=\"\\t\")\n\n# gc.collect()","metadata":{"papermill":{"duration":0.20196,"end_time":"2023-07-15T21:55:46.717784","exception":false,"start_time":"2023-07-15T21:55:46.515824","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Show some statistics on submitted data ","metadata":{"papermill":{"duration":0.179787,"end_time":"2023-07-15T21:55:47.07847","exception":false,"start_time":"2023-07-15T21:55:46.898683","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\nif (mode_submit is not None) and ( mode_submit != False ): # None\n v = df_finalSubmission['Prediction']\n display( v.describe() )\n plt.hist(v, bins = 100 )\n plt.show()\n for t in [0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1]:\n s = (v > t).sum()\n# print(t,m.sum(),m.sum()/(Y_submit.shape[0] * Y_submit.shape[1] ))\n print(t,s,s/(len(v) )) \n \n gc.collect() ","metadata":{"papermill":{"duration":0.193431,"end_time":"2023-07-15T21:55:47.455369","exception":false,"start_time":"2023-07-15T21:55:47.261938","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.180442,"end_time":"2023-07-15T21:55:47.817731","exception":false,"start_time":"2023-07-15T21:55:47.637289","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.178461,"end_time":"2023-07-15T21:55:48.175042","exception":false,"start_time":"2023-07-15T21:55:47.996581","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('%.1f seconds passed total '%(time.time()-t0start) )\n","metadata":{"papermill":{"duration":0.190137,"end_time":"2023-07-15T21:55:48.545039","exception":false,"start_time":"2023-07-15T21:55:48.354902","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.183769,"end_time":"2023-07-15T21:55:48.908346","exception":false,"start_time":"2023-07-15T21:55:48.724577","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}