{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"How to validate Alexander's solution https://www.kaggle.com/code/alexandervc/baseline-multilabel-to-multitarget-binary using the metric code provided by the organizers.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"# Train - Test split\n","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nimport pandas as pd\ntrainTerms = pd.read_csv('/kaggle/input/cafa-5-protein-function-prediction/Train/train_terms.tsv',\n                         sep=\"\\t\")\nX = trainTerms.EntryID.unique().tolist()\nIX_train, IX_test, _,_ = train_test_split(X, X, train_size=0.8, random_state=42)\n\ntrainTerms.loc[trainTerms.EntryID.isin(IX_train)].to_csv('train.tsv', \n                                                        sep='\\t', index=False)\ntrainTerms.loc[trainTerms.EntryID.isin(IX_test)].to_csv('valid.tsv', \n                                                        sep='\\t', index=False)\n\nlen(IX_train), len(IX_test)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Metric code","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport multiprocessing as mp\nimport copy\nimport logging\n\nclass Graph:\n    \"\"\"\n    Ontology class. One ontology == one namespace\n    DAG is the adjacence matrix (sparse) which represent a Directed Acyclic Graph where\n    DAG(i,j) == 1 means that the go term i is_a (or is part_of) j\n    Parents that are in a different namespace are discarded\n    \"\"\"\n    def __init__(self, namespace, terms_dict, ia_dict=None, orphans=False):\n        \"\"\"\n        terms_dict = {term: {name: , namespace: , def: , alt_id: , rel:}}\n        \"\"\"\n        self.namespace = namespace\n        self.dag = []  # [[], ...] terms (rows, axis 0) x parents (columns, axis 1)\n        self.terms_dict = {}  # {term: {index: , name: , namespace: , def: }  used to assign term indexes in the gt\n        self.terms_list = []  # [{id: term, name:, namespace: , def:, adg: [], children: []}, ...]\n        self.idxs = None  # Number of terms\n        self.order = None\n        self.toi = None\n        self.ia = None\n\n        rel_list = []\n        for self.idxs, (term_id, term) in enumerate(terms_dict.items()):\n            rel_list.extend([[term_id, rel, term['namespace']] for rel in term['rel']])\n            self.terms_list.append({'id': term_id, 'name': term['name'], 'namespace': namespace, 'def': term['def'],\n                                 'adj': [], 'children': []})\n            self.terms_dict[term_id] = {'index': self.idxs, 'name': term['name'], 'namespace': namespace, 'def': term['def']}\n            for a_id in term['alt_id']:\n                self.terms_dict[a_id] = copy.copy(self.terms_dict[term_id])\n        self.idxs += 1\n\n        self.dag = np.zeros((self.idxs, self.idxs), dtype='bool')\n\n        # id1 term (row, axis 0), id2 parent (column, axis 1)\n        for id1, id2, ns in rel_list:\n            if self.terms_dict.get(id2):\n                i = self.terms_dict[id1]['index']\n                j = self.terms_dict[id2]['index']\n                self.dag[i, j] = 1\n                self.terms_list[i]['adj'].append(j)\n                self.terms_list[j]['children'].append(i)\n                logging.debug(\"i,j {},{} {},{}\".format(i, j, id1, id2))\n            else:\n                logging.debug('Skipping branch to external namespace: {}'.format(id2))\n        logging.debug(\"dag {}\".format(self.dag))\n        # Topological sorting\n        self.top_sort()\n        logging.debug(\"order sorted {}\".format(self.order))\n\n        if orphans:\n            self.toi = np.arange(self.dag.shape[0])  # All terms, also those without parents\n        else:\n            self.toi = np.nonzero(self.dag.sum(axis=1) > 0)[0]  # Only terms with parents\n        logging.debug(\"toi {}\".format(self.toi))\n\n        if ia_dict is not None:\n            self.set_ia(ia_dict)\n\n        return\n\n    def top_sort(self):\n        \"\"\"\n        Takes a sparse matrix representing a DAG and returns an array with nodes indexes in topological order\n        https://en.wikipedia.org/wiki/Topological_sorting\n        \"\"\"\n        indexes = []\n        visited = 0\n        (rows, cols) = self.dag.shape\n\n        # create a vector containing the in-degree of each node\n        in_degree = self.dag.sum(axis=0)\n        # logging.debug(\"degree {}\".format(in_degree))\n\n        # find the nodes with in-degree 0 (leaves) and add them to the queue\n        queue = np.nonzero(in_degree == 0)[0].tolist()\n        # logging.debug(\"queue {}\".format(queue))\n\n        # for each element of the queue increment visits, add them to the list of ordered nodes\n        # and decrease the in-degree of the neighbor nodes\n        # and add them to the queue if they reach in-degree == 0\n        while queue:\n            visited += 1\n            idx = queue.pop(0)\n            indexes.append(idx)\n            in_degree[idx] -= 1\n            l = self.terms_list[idx]['adj']\n            if len(l) > 0:\n                for j in l:\n                    in_degree[j] -= 1\n                    if in_degree[j] == 0:\n                        queue.append(j)\n\n        # if visited is equal to the number of nodes in the graph then the sorting is complete\n        # otherwise the graph can't be sorted with topological order\n        if visited == rows:\n            self.order = indexes\n        else:\n            raise Exception(\"The sparse matrix doesn't represent an acyclic graph\")\n\n    def set_ia(self, ia_dict):\n        self.ia = np.zeros(self.idxs, dtype='float')\n        for term_id in self.terms_dict:\n            if ia_dict.get(term_id):\n                self.ia[self.terms_dict[term_id]['index']] = ia_dict.get(term_id)\n            else:\n                logging.debug('Missing IA for term: {}'.format(term_id))\n        # Convert inf to zero\n        np.nan_to_num(self.ia, copy=False, nan=0, posinf=0, neginf=0)\n        self.toi = np.nonzero(self.ia > 0)[0]\n\n\nclass Prediction:\n    \"\"\"\n    The score matrix contains the scores given by the predictor for every node of the ontology\n    \"\"\"\n    def __init__(self, ids, matrix, idx, namespace=None):\n        self.ids = ids\n        self.matrix = matrix  # scores\n        self.next_idx = idx\n        # self.n_pred_seq = idx + 1\n        self.namespace = namespace\n\n    def __str__(self):\n        return \"\\n\".join([\"{}\\t{}\\t{}\".format(index, self.matrix[index], self.namespace) for index, _id in enumerate(self.ids)])\n\n\nclass GroundTruth:\n    def __init__(self, ids, matrix, namespace=None):\n        self.ids = ids\n        self.matrix = matrix\n        self.namespace = namespace\n\n\ndef propagate(matrix, ont, order, mode='max'):\n    \"\"\"\n    Update inplace the score matrix (proteins x terms) up to the root taking the max between children and parents\n    \"\"\"\n    if matrix.shape[0] == 0:\n        raise Exception(\"Empty matrix\")\n\n    deepest = np.where(np.sum(matrix[:, order], axis=0) > 0)[0][0]\n    if deepest.size == 0:\n        raise Exception(\"The matrix is empty\")\n\n    # Remove leaves\n    order_ = np.delete(order, [range(0, deepest)])\n\n    for i in order_:\n        # Get direct children\n        children = np.where(ont.dag[:, i] != 0)[0]\n        if children.size > 0:\n            cols = np.concatenate((children, [i]))\n            if mode == 'max':\n                matrix[:, i] = matrix[:, cols].max(axis=1)\n            elif mode == 'fill':\n                rows = np.where(matrix[:, i] == 0)[0]\n                if rows.size:\n                    idx = np.ix_(rows, cols)\n                    matrix[rows, i] = matrix[idx].max(axis=1)[0]\n    return\n\n\ndef obo_parser(obo_file, valid_rel=(\"is_a\", \"part_of\")):\n    \"\"\"\n    Parse a OBO file and returns a list of ontologies, one for each namespace.\n    Obsolete terms are excluded as well as external namespaces.\n    \"\"\"\n    term_dict = {}\n    term_id = None\n    namespace = None\n    name = None\n    term_def = None\n    alt_id = []\n    rel = []\n    obsolete = True\n    with open(obo_file) as f:\n        for line in f:\n            line = line.strip().split(\": \")\n            if line and len(line) > 1:\n                k = line[0]\n                v = \": \".join(line[1:])\n                if k == \"id\":\n                    # Populate the dictionary with the previous entry\n                    if term_id is not None and obsolete is False and namespace is not None:\n                        term_dict.setdefault(namespace, {})[term_id] = {'name': name,\n                                                                       'namespace': namespace,\n                                                                       'def': term_def,\n                                                                       'alt_id': alt_id,\n                                                                       'rel': rel}\n                    # Assign current term ID\n                    term_id = v\n\n                    # Reset optional fields\n                    alt_id = []\n                    rel = []\n                    obsolete = False\n                    namespace = None\n\n                elif k == \"alt_id\":\n                    alt_id.append(v)\n                elif k == \"name\":\n                    name = v\n                elif k == \"namespace\" and v != 'external':\n                    namespace = v\n                elif k == \"def\":\n                    term_def = v\n                elif k == 'is_obsolete':\n                    obsolete = True\n                elif k == \"is_a\" and k in valid_rel:\n                    s = v.split('!')[0].strip()\n                    rel.append(s)\n                elif k == \"relationship\" and v.startswith(\"part_of\") and \"part_of\" in valid_rel:\n                    s = v.split()[1].strip()\n                    rel.append(s)\n\n        # Last record\n        if obsolete is False and namespace is not None:\n            term_dict.setdefault(namespace, {})[term_id] = {'name': name,\n                                                          'namespace': namespace,\n                                                          'def': term_def,\n                                                          'alt_id': alt_id,\n                                                          'rel': rel}\n    return term_dict\n\n\ndef gt_parser(gt_file, ontologies):\n    \"\"\"\n    Parse ground truth file. Discard terms not included in the ontology.\n    \"\"\"\n    gt_dict = {}\n    with open(gt_file) as f:\n        for line in f:\n            line = line.strip().split()\n            if line:\n                p_id, term_id = line[:2]\n                for ont in ontologies:\n                    if term_id in ont.terms_dict:\n                        gt_dict.setdefault(ont.namespace, {}).setdefault(p_id, []).append(term_id)\n                        break\n\n    gts = {}\n    for ont in ontologies:\n        if gt_dict.get(ont.namespace):\n            matrix = np.zeros((len(gt_dict[ont.namespace]), ont.idxs), dtype='bool')\n            ids = {}\n            for i, p_id in enumerate(gt_dict[ont.namespace]):\n                ids[p_id] = i\n                for term_id in gt_dict[ont.namespace][p_id]:\n                    matrix[i, ont.terms_dict[term_id]['index']] = 1\n            propagate(matrix, ont, ont.order, mode='max')\n            gts[ont.namespace] = GroundTruth(ids, matrix, ont.namespace)\n\n    return gts\n\n\ndef pred_parser(f, ontologies, gts, prop_mode, max_terms=None):\n    \"\"\"\n    Parse a prediction file and returns a list of prediction objects, one for each namespace.\n    If a predicted is predicted multiple times for the same target, it stores the max.\n    This is the slow step if the input file is huge, ca. 1 minute for 5GB input on SSD disk.\n    \"\"\"\n    ids = {}\n    matrix = {}\n    ns_dict = {}  # {namespace: term}\n    onts = {ont.namespace: ont for ont in ontologies}\n    for ns in gts:\n        matrix[ns] = np.zeros(gts[ns].matrix.shape, dtype='float')\n        ids[ns] = {}\n        for term in onts[ns].terms_dict:\n            ns_dict[term] = ns\n\n    for line in f:\n        p_id, term_id, prob = line\n        ns = ns_dict.get(term_id)\n        if ns in gts and p_id in gts[ns].ids:\n            i = gts[ns].ids[p_id]\n            if max_terms is None or np.count_nonzero(matrix[ns][i]) <= max_terms:\n                j = onts[ns].terms_dict.get(term_id)['index']\n                ids[ns][p_id] = i\n                matrix[ns][i, j] = max(matrix[ns][i, j], float(prob))\n\n    predictions = []\n    for ns in ids:\n        if ids[ns]:\n            propagate(matrix[ns], onts[ns], onts[ns].order, mode=prop_mode)\n            predictions.append(Prediction(ids[ns], matrix[ns], len(ids[ns]), ns))\n\n    if not predictions:\n        raise Exception(\"Empty prediction, check format\")\n\n    return predictions\n\n\ndef ia_parser(file):\n    ia_dict = {}\n    with open(file) as f:\n        for line in f:\n            if line:\n                term, ia = line.strip().split()\n                ia_dict[term] = float(ia)\n    return ia_dict\n\n# Computes the root terms in the dag\ndef get_roots_idx(dag):\n    return np.where(dag.sum(axis=1) == 0)[0]\n\n\n# Computes the leaf terms in the dag\ndef get_leafs_idx(dag):\n    return np.where(dag.sum(axis=0) == 0)[0]\n\n\n# Return a mask for all the predictions (matrix) >= tau\ndef solidify_prediction(pred, tau):\n    return pred >= tau\n\n\n# computes the f metric for each precision and recall in the input arrays\ndef compute_f(pr, rc):\n    n = 2 * pr * rc\n    d = pr + rc\n    return np.divide(n, d, out=np.zeros_like(n, dtype=float), where=d != 0)\n\n\ndef compute_s(ru, mi):\n    return np.sqrt(ru**2 + mi**2)\n    # return np.where(np.isnan(ru), mi, np.sqrt(ru + np.nan_to_num(mi)))\n\n\ndef compute_metrics_(tau_arr, g, pred, toi, n_gt, wn_gt=None, ic_arr=None):\n\n    metrics = np.zeros((len(tau_arr), 7), dtype='float')  # cov, pr, rc, wpr, wrc, ru, mi\n\n    for i, tau in enumerate(tau_arr):\n\n        p = solidify_prediction(pred.matrix[:, toi], tau)\n\n        # number of proteins with at least one term predicted with score >= tau\n        metrics[i, 0] = (p.sum(axis=1) > 0).sum()\n\n        # Terms subsets\n        intersection = np.logical_and(p, g)  # TP\n\n        # Subsets size\n        n_pred = p.sum(axis=1)\n        n_intersection = intersection.sum(axis=1)\n\n        # Precision, recall\n        metrics[i, 1] = np.divide(n_intersection, n_pred, out=np.zeros_like(n_intersection, dtype='float'),\n                                  where=n_pred > 0).sum()\n        metrics[i, 2] = np.divide(n_intersection, n_gt, out=np.zeros_like(n_gt, dtype='float'), where=n_gt > 0).sum()\n\n        if ic_arr is not None:\n            # Terms subsets\n            remaining = np.logical_and(np.logical_not(p), g)  # FN --> not predicted but in the ground truth\n            mis = np.logical_and(p, np.logical_not(g))  # FP --> predicted but not in the ground truth\n\n            # Weighted precision, recall\n            wn_pred = (p * ic_arr[toi]).sum(axis=1)\n            wn_intersection = (intersection * ic_arr[toi]).sum(axis=1)\n\n            metrics[i, 3] = np.divide(wn_intersection, wn_pred, out=np.zeros_like(n_intersection, dtype='float'),\n                                      where=n_pred > 0).sum()\n            metrics[i, 4] = np.divide(wn_intersection, wn_gt, out=np.zeros_like(n_intersection, dtype='float'),\n                                      where=n_gt > 0).sum()\n\n            # Misinformation, remaining uncertainty\n            metrics[i, 5] = (remaining * ic_arr[toi]).sum(axis=1).sum()\n            metrics[i, 6] = (mis * ic_arr[toi]).sum(axis=1).sum()\n    return metrics\n\n\ndef compute_metrics(pred, gt, toi, tau_arr, ic_arr=None, n_cpu=0):\n    \"\"\"\n    Takes the prediction and the ground truth and for each threshold in tau_arr\n    calculates the confusion matrix and returns the coverage,\n    precision, recall, remaining uncertainty and misinformation.\n    Toi is the list of terms (indexes) to be considered\n    \"\"\"\n    g = gt.matrix[:, toi]\n    n_gt = g.sum(axis=1)\n    wn_gt = None\n    if ic_arr is not None:\n        wn_gt = (g * ic_arr[toi]).sum(axis=1)\n\n    # Parallelization\n    if n_cpu == 0:\n        n_cpu = mp.cpu_count()\n\n    arg_lists = [[tau_arr, g, pred, toi, n_gt, wn_gt, ic_arr] for tau_arr in np.array_split(tau_arr, n_cpu)]\n    if 0:\n        # Original parallel way (# It does not work on Kaggle)\n        arg_lists = [[tau_arr, g, pred, toi, n_gt, wn_gt, ic_arr] for tau_arr in np.array_split(tau_arr, n_cpu)]\n        with mp.Pool(processes=n_cpu) as pool:\n            metrics = np.concatenate(pool.starmap(compute_metrics_, arg_lists), axis=0)\n    else: \n        # no-parallel: \n        metrics = compute_metrics_(tau_arr, g, pred, toi, n_gt, wn_gt, ic_arr )\n\n    return pd.DataFrame(metrics, columns=[\"cov\", \"pr\", \"rc\", \"wpr\", \"wrc\", \"ru\", \"mi\"])\n\n\ndef evaluate_prediction(prediction, gt, ontologies, tau_arr, normalization='cafa', n_cpu=0):\n    dfs = []\n    for p in prediction:\n        ns = p.namespace\n        ne = np.full(len(tau_arr), gt[ns].matrix.shape[0])\n\n        ont = [o for o in ontologies if o.namespace == ns][0]\n\n        # cov, pr, rc, wpr, wrc, ru, mi\n        metrics = compute_metrics(p, gt[ns], ont.toi, tau_arr, ont.ia, n_cpu)\n\n        for column in [\"pr\", \"rc\", \"wpr\", \"wrc\", \"ru\", \"mi\"]:\n            if normalization == 'gt' or (column in [\"rc\", \"wrc\"] and normalization == 'cafa'):\n                metrics[column] = np.divide(metrics[column], ne, out=np.zeros_like(metrics[column], dtype='float'), where=ne > 0)\n            else:\n                metrics[column] = np.divide(metrics[column], metrics[\"cov\"], out=np.zeros_like(metrics[column], dtype='float'), where=metrics[\"cov\"] > 0)\n\n        metrics['ns'] = [ns] * len(tau_arr)\n        metrics['tau'] = tau_arr\n        metrics['cov'] = np.divide(metrics['cov'], ne, out=np.zeros_like(metrics['cov'], dtype='float'), where=ne > 0)\n        metrics['f'] = compute_f(metrics['pr'], metrics['rc'])\n        metrics['wf'] = compute_f(metrics['wpr'], metrics['wrc'])\n        metrics['s'] = compute_s(metrics['ru'], metrics['mi'])\n\n        dfs.append(metrics)\n\n    return pd.concat(dfs)","metadata":{"execution":{"iopub.status.busy":"2023-04-24T19:01:59.651196Z","iopub.execute_input":"2023-04-24T19:01:59.651677Z","iopub.status.idle":"2023-04-24T19:01:59.754554Z","shell.execute_reply.started":"2023-04-24T19:01:59.651631Z","shell.execute_reply":"2023-04-24T19:01:59.753442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Tau array, used to compute metrics at different score thresholds\nth_step = 0.01\ntau_arr = np.arange(0.01, 1, th_step)\n#Consider terms without parents, e.g. the root(s), in the evaluation\nno_orphans = False\n# Parse and set information accretion (optional)\nia_dict = ia_parser('/kaggle/input/cafa-5-protein-function-prediction/IA.txt')\n\n# Parse the OBO file and creates a different graph for each namespace\nontologies = []\nobo_file = '/kaggle/input/cafa-5-protein-function-prediction/Train/go-basic.obo'\nfor ns, terms_dict in obo_parser(obo_file).items():\n    ontologies.append(Graph(ns, terms_dict, ia_dict, not no_orphans))\n    \ngt = gt_parser('valid.tsv', ontologies)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sample metric evaluation","metadata":{}},{"cell_type":"code","source":"import tqdm\ntt = pd.read_csv('valid.tsv', sep='\\t')\n\npreds = []\nfor i in tqdm.tqdm(range(tt.shape[0])):\n    if np.random.rand() > 0.1:\n        preds.append((tt.EntryID.values[i], tt.term.values[i], np.random.rand()))\n\npreds = pred_parser(preds, ontologies, gt, prop_mode='fill', max_terms=500)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_pred = evaluate_prediction(preds, gt, ontologies, tau_arr, n_cpu=1)\ndf_pred","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_pred.groupby('ns').agg({'f':'max'})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ridge validation","metadata":{}},{"cell_type":"code","source":"n_labels_to_consider = 300\ntrainTerms = pd.read_csv(\"train.tsv\",sep=\"\\t\")\ntrain_ids_set = set(trainTerms.EntryID.unique().tolist())\nprint(trainTerms.shape)\ndisplay(trainTerms.head(2))\nvec_freqCount = (trainTerms['term'].value_counts())\nprint(vec_freqCount )\n\nprint()\nlabels_to_consider = list(vec_freqCount.index[:n_labels_to_consider] )\nprint('n_labels_to_consider:', len(labels_to_consider), 'First 10:', labels_to_consider[:10] ) \n\nfn = '/kaggle/input/t5embeds/train_ids.npy'\nvec_train_protein_ids = np.load(fn)\nt_ids = vec_train_protein_ids.copy()\nvec_train_protein_ids = [(x in train_ids_set) for x in vec_train_protein_ids]\nfn = '/kaggle/input/t5embeds/train_embeds.npy'\nX = np.load(fn)\n\nX_valid = X[~np.array(vec_train_protein_ids)]\nvec_test_protein_ids = t_ids[~np.array(vec_train_protein_ids)]\n\nX = X[vec_train_protein_ids]\nvec_train_protein_ids = t_ids[vec_train_protein_ids]\nprint(vec_train_protein_ids.shape)\n\ntrain_size = trainTerms.EntryID.nunique()\nY = np.zeros( (train_size, n_labels_to_consider) )\nprint(Y.shape)\n\nseries_train_protein_ids = pd.Series(vec_train_protein_ids ) # \n\ntrainTerms_smaller = trainTerms[ trainTerms['term'].isin( labels_to_consider ) ] # to speed-up the next step \nprint( trainTerms_smaller.shape)\n\nfor i in range(Y.shape[1]):\n    m = trainTerms_smaller['term'] ==  labels_to_consider[i]\n    Y[:,i] =  series_train_protein_ids.isin(  set(trainTerms_smaller[m]['EntryID'] ) ).astype(float )\n    if (i % 10) == 0: \n        print(i, m.sum())\nY ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import Ridge\n\nmodel = Ridge(alpha=1.0)\nstr_model_id = 'Ridge1'\n\ndf_models_stat = pd.DataFrame()\nmodel","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nfrom sklearn.metrics import roc_auc_score\n\nt0 = time.time()\nmodel.fit(X,Y)\nY_pred_test = model.predict(X_valid)\npreds = []\nfor i in range(len(vec_test_protein_ids)):\n    for j in range(len(labels_to_consider)):\n        if Y_pred_test[i,j] >= 0.01:\n            preds.append((vec_test_protein_ids[i], \n                          labels_to_consider[j],\n                          Y_pred_test[i,j]\n                         ))\nprint(len(preds))\npreds = pred_parser(preds, ontologies, gt, prop_mode='fill', max_terms=None)\ndf_metrics = evaluate_prediction(preds, gt, ontologies, tau_arr, n_cpu=1)\ndf_metrics","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_metrics.groupby('ns').agg({'wf':'max'}).mean())\ndf_metrics.groupby('ns').agg({'wf':'max'})","metadata":{},"execution_count":null,"outputs":[]}]}