{"cells":[{"metadata":{},"cell_type":"markdown","source":"For those who uses TensorFlow 2.1.0 I want to share the Quadratic Weighted Kappa metric implementation used in the [Prostate cANcer graDe Assessment (PANDA) Challenge](https://www.kaggle.com/c/prostate-cancer-grade-assessment/overview/evaluation)."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\n\nclass QuadraticWeightedKappa(tf.keras.metrics.Metric):\n    def __init__(self, maxClassesCount=6, name='Kappa', **kwargs):        \n        super(QuadraticWeightedKappa, self).__init__(name=name, **kwargs)\n        self.M = maxClassesCount\n\n        self.O = self.add_weight(name='O', initializer='zeros',shape=(self.M,self.M,), dtype=tf.int64)\n        self.W = self.add_weight(name='W', initializer='zeros',shape=(self.M,self.M,), dtype=tf.float32)\n        self.actualHist = self.add_weight(name='actHist', initializer='zeros',shape=(self.M,), dtype=tf.int64)\n        self.predictedHist = self.add_weight(name='predHist', initializer='zeros',shape=(self.M,), dtype=tf.int64)\n        \n        # filling up the content of W once\n        w = np.zeros((self.M,self.M),dtype=np.float32)\n        for i in range(0,self.M):\n            for j in range(0,self.M):\n                w[i,j] = (i-j)*(i-j) / ((self.M - 1)*(self.M - 1))\n        self.W.assign(w)\n    \n    def reset_states(self):\n        \"\"\"Resets all of the metric state variables.\n        This function is called between epochs/steps,\n        when a metric is evaluated during training.\n        \"\"\"\n        # value should be a Numpy array\n        zeros1D = np.zeros(self.M)\n        zeros2D = np.zeros((self.M,self.M))\n        tf.keras.backend.batch_set_value([\n            (self.O, zeros2D),\n            (self.actualHist, zeros1D),\n            (self.predictedHist,zeros1D)\n        ])\n\n\n\n    def update_state(self, y_true, y_pred, sample_weight=None):\n        # shape is: Batch x 1\n        y_true = tf.reshape(y_true, [-1])\n        y_pred = tf.reshape(y_pred, [-1])\n\n        y_true_int = tf.cast(tf.math.round(y_true), dtype=tf.int64)\n        y_pred_int = tf.cast(tf.math.round(y_pred), dtype=tf.int64)\n\n        confM = tf.math.confusion_matrix(y_true_int, y_pred_int, dtype=tf.int64, num_classes=self.M)\n\n        # incremeting confusion matrix and standalone histograms\n        self.O.assign_add(confM)\n\n        cur_act_hist = tf.math.reduce_sum(confM, 0)\n        self.actualHist.assign_add(cur_act_hist)\n\n        cur_pred_hist = tf.math.reduce_sum(confM, 1)\n        self.predictedHist.assign_add(cur_pred_hist)\n\n    def result(self):\n        EFloat = tf.cast(tf.tensordot(self.actualHist,self.predictedHist, axes=0),dtype=tf.float32)\n        OFloat = tf.cast(self.O,dtype=tf.float32)\n        \n        # E must be normalized \"such that E and O have the same sum\"\n        ENormalizedFloat = EFloat / tf.math.reduce_sum(EFloat) * tf.math.reduce_sum(OFloat)\n\n        \n        return 1.0 - tf.math.reduce_sum(tf.math.multiply(self.W, OFloat))/tf.math.reduce_sum(tf.multiply(self.W, ENormalizedFloat))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"To use the metric in TF Keras API use it as follows in the model compile call:\n```\nmodel.compile(\n          optimizer= ... ,\n          loss= ... ,\n          metrics=[QuadraticWeightedKappa()]\n          )\n```\n\nI suppose it can be easily adobted for pure Keras.\n\nHope, it will be helpful :-)"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}