{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook is from @yyama, I think there is an error in that notebooks final metric calculation, I have fixed it below.\n\nReference: https://www.kaggle.com/competitions/rsna-2022-cervical-spine-fracture-detection/discussion/341854","metadata":{}},{"cell_type":"markdown","source":"In the discussion, the division was done by the sum of the weights, but isn't it correct to divide by each row?\n\nver 4: @harshitsheoran pointed out mistake, fixed!","metadata":{}},{"cell_type":"code","source":"#@solverworld added: I have fixed the loss calculation at the bottom\n#competiton_loss_row_norm2()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn as nn","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-26T20:37:44.816467Z","iopub.execute_input":"2022-09-26T20:37:44.817208Z","iopub.status.idle":"2022-09-26T20:37:47.06703Z","shell.execute_reply.started":"2022-09-26T20:37:44.817113Z","shell.execute_reply":"2022-09-26T20:37:47.065704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/rsna-2022-cervical-spine-fracture-detection/train.csv')\ntargets = ['patient_overall', 'C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7']\nlabels = torch.tensor(train[targets].values)\nmean_values = train.mean(axis=0).values\nmean_values = torch.tensor(np.vstack([mean_values]*len(train)))","metadata":{"execution":{"iopub.status.busy":"2022-09-26T20:37:47.069379Z","iopub.execute_input":"2022-09-26T20:37:47.070891Z","iopub.status.idle":"2022-09-26T20:37:47.115206Z","shell.execute_reply.started":"2022-09-26T20:37:47.070836Z","shell.execute_reply":"2022-09-26T20:37:47.113005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This metrics from https://www.kaggle.com/competitions/rsna-2022-cervical-spine-fracture-detection/discussion/341854","metadata":{}},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n# change it to nn.BCELoss(reduction='none') if you have sigmoid activation in last layer\nloss_fn = nn.BCELoss(reduction=\"none\") \n\ncompetition_weights = {\n    '-' : torch.tensor([7, 1, 1, 1, 1, 1, 1, 1], dtype=torch.float, device=device),\n    '+' : torch.tensor([14, 2, 2, 2, 2, 2, 2, 2], dtype=torch.float, device=device),\n}\n","metadata":{"execution":{"iopub.status.busy":"2022-09-26T20:37:47.120183Z","iopub.execute_input":"2022-09-26T20:37:47.120968Z","iopub.status.idle":"2022-09-26T20:37:47.132167Z","shell.execute_reply.started":"2022-09-26T20:37:47.120895Z","shell.execute_reply":"2022-09-26T20:37:47.130933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# original metrics from the discussion https://www.kaggle.com/competitions/rsna-2022-cervical-spine-fracture-detection/discussion/341854\ndef competiton_loss(y_hat, y):\n    loss = loss_fn(y_hat, y)\n    weights = y * competition_weights['+'] + (1 - y) * competition_weights['-']\n    loss = (loss * weights).sum(axis=1).mean()\n\n    return loss / weights.sum()\n\nprint(competiton_loss(mean_values, labels.double()))","metadata":{"execution":{"iopub.status.busy":"2022-09-26T20:37:47.134945Z","iopub.execute_input":"2022-09-26T20:37:47.135602Z","iopub.status.idle":"2022-09-26T20:37:47.152603Z","shell.execute_reply.started":"2022-09-26T20:37:47.135565Z","shell.execute_reply":"2022-09-26T20:37:47.151435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Too small value?","metadata":{}},{"cell_type":"code","source":"# without weights normalization\ndef competiton_loss_wo_norm(y_hat, y):\n    loss = loss_fn(y_hat, y)\n    weights = y * competition_weights['+'] + (1 - y) * competition_weights['-']\n    loss = (loss * weights).sum(axis=1).mean()\n\n    return loss #/ weights.sum()\n\nprint(competiton_loss_wo_norm(mean_values, labels.double()))","metadata":{"execution":{"iopub.status.busy":"2022-09-26T20:37:47.154512Z","iopub.execute_input":"2022-09-26T20:37:47.156548Z","iopub.status.idle":"2022-09-26T20:37:47.166511Z","shell.execute_reply.started":"2022-09-26T20:37:47.156507Z","shell.execute_reply":"2022-09-26T20:37:47.165367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Too big value?","metadata":{}},{"cell_type":"code","source":"# with row-wise weights normalization\ndef competiton_loss_row_norm(y_hat, y):\n    loss = loss_fn(y_hat, y)\n    weights = y * competition_weights['+'] + (1 - y) * competition_weights['-']\n    loss = (loss * weights).sum(axis=1)\n    w_sum = weights.sum(axis=1)\n    loss = torch.div(loss, w_sum)\n    return loss.mean()\n\nprint(\"The overall loss is\",competiton_loss_row_norm(mean_values, labels.double()))","metadata":{"execution":{"iopub.status.busy":"2022-09-26T20:37:47.168229Z","iopub.execute_input":"2022-09-26T20:37:47.168946Z","iopub.status.idle":"2022-09-26T20:37:47.184334Z","shell.execute_reply.started":"2022-09-26T20:37:47.168911Z","shell.execute_reply":"2022-09-26T20:37:47.183159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#cannot divide each row by the weights of that row, that effectively \n#removes weighting of pos samples","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Fixed\ndef competiton_loss_row_norm2(y_hat, y):\n    loss = loss_fn(y_hat, y)\n    weights = y * competition_weights['+'] + (1 - y) * competition_weights['-']\n    loss = (loss * weights).sum()\n    w_sum = weights.sum()\n    loss = torch.div(loss, w_sum)\n    return loss\n\nprint(\"The overall loss 2 is\",competiton_loss_row_norm2(mean_values, labels.double()))","metadata":{"execution":{"iopub.status.busy":"2022-09-26T20:39:24.362626Z","iopub.execute_input":"2022-09-26T20:39:24.36301Z","iopub.status.idle":"2022-09-26T20:39:24.373951Z","shell.execute_reply.started":"2022-09-26T20:39:24.36298Z","shell.execute_reply":"2022-09-26T20:39:24.372595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Need to curve each means to make it the best due to the 2x weighting on pos samples\ncurved=mean_values*(2/(1+mean_values))\nprint(\"The improved overall loss is\",competiton_loss_row_norm2(curved, labels.double()))","metadata":{"execution":{"iopub.status.busy":"2022-09-26T20:42:10.153594Z","iopub.execute_input":"2022-09-26T20:42:10.154009Z","iopub.status.idle":"2022-09-26T20:42:10.163833Z","shell.execute_reply.started":"2022-09-26T20:42:10.153976Z","shell.execute_reply":"2022-09-26T20:42:10.162524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" mean preds can get 0.57 on public LB, which is almost identical to this value.","metadata":{}}]}