{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# First attempt at an end to end submission!\nfrom datetime import datetime\nimport os\nimport pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TIME_EVENTS = [('Book Start', datetime.now())]\nprint(f'Start of Book: {TIME_EVENTS[0][1]}')\ndef runtime(desc):\n    global TIME_EVENTS\n    now = datetime.now()\n    TIME_EVENTS.append((desc, now))\n    print(f'Now: {desc}: {now}')\n    print(f'Time Since First Event, {TIME_EVENTS[0][0]}: {TIME_EVENTS[-1][1]-TIME_EVENTS[0][1]})')\n    print(f'Time Since Last Event, {TIME_EVENTS[-2][0]}: {TIME_EVENTS[-1][1]-TIME_EVENTS[-2][1]}')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"if os.path.exists('/kaggle/'):\n    !pip install ../input/fastai2-wheels/fastscript-0.1.4-py3-none-any.whl > /dev/null\n    !pip install ../input/fastai2-wheels/kornia-0.2.0-py2.py3-none-any.whl > /dev/null\n    !pip install ../input/fastai2-wheels/nbdev-0.2.12-py3-none-any.whl > /dev/null\n    !pip install ../input/fastai2-wheels/fastprogress-0.2.3-py3-none-any.whl > /dev/null\n    !pip install ../input/fastai2-wheels/fastcore-0.1.16-py3-none-any.whl > /dev/null\n    !pip install ../input/fastai2-wheels/fastai2-0.0.16-py3-none-any.whl > /dev/null\n    \n    !mkdir -p /root/.cache/torch/checkpoints/\n    !cp '../input/pytorch-pretrained-models/resnet34-333f7ec4.pth' '/root/.cache/torch/checkpoints/resnet34-333f7ec4.pth'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Bug in fastai2-0.0.16: https://github.com/fastai/fastai2/issues/331\nimport subprocess\nc = subprocess.call(\n    ['sed',\n     's/bias_std=0)/bias_std=0.01)/',\n     '-i',\n     '/opt/conda/lib/python3.7/site-packages/fastai2/layers.py'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Approximately 3 minutes 45 seconds\nruntime('Pip Installs')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import openslide\n\nfrom PIL import Image\nimport pandas as pd\nimport numpy as np\nimport torch\nfrom fastai2.vision.all import *\ntorch.cuda.set_device(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED=182\n\n\n# TRAIN_DIR = 'test_set_2'\n# TRAIN_DIR = '/opt/mount/train_images'\nTRAIN_DIR = '../input/prostate-cancer-grade-assessment/train_images'\n# MASK_DIR = 'test_set_2_masks'\n# MASK_DIR = '/opt/mount/train_label_masks'\nMASK_DIR = '../input/prostate-cancer-grade-assessment/train_label_masks'\nTEST_DIR = '../input/prostate-cancer-grade-assessment/test_images'\n\n_MD = set([y.replace('_mask', '') for y in os.listdir(MASK_DIR)])\nTRAIN_IDS = [os.path.join(TRAIN_DIR, x) for x in os.listdir(TRAIN_DIR) if x in _MD]\nprint(len(TRAIN_IDS))\nTARGET_DIM = 128\n\nRAD_CODES = [\n    'background', 'healthy stroma', 'healthy epithelium',\n    'gleason level 3', 'gleason level 4', 'gleason level 5']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _open_and_resize(path):\n    slide = openslide.OpenSlide(path)\n    l2_dims = slide.level_dimensions[2][:2]\n\n    return np.array(slide.read_region(\n        (0,0), 2, l2_dims).resize(\n            (TARGET_DIM,TARGET_DIM)))[:,:,:3]\n\n\ndef get_x(path):\n    return _open_and_resize(path)\n\n\ndef get_y(path):\n    name = os.path.basename(path)\n    path = os.path.join(MASK_DIR, name).replace('.tiff', '_mask.tiff')\n    return _open_and_resize(path)\n\n\ndef create_datablock(bs=16):\n    # Rad / downsampled only for the time being\n\n    dblock = DataBlock(\n        blocks=(ImageBlock, MaskBlock(codes=['bg', 'good s', 'good e', 'g3', 'g4', 'g5'])),\n        splitter=RandomSplitter(0.2, seed=SEED),\n        get_x=get_x,\n        get_y=get_y,\n    )\n    res = DataLoaders.from_dblock(dblock, TRAIN_IDS, bs=bs)\n    return dblock, res\n\n\ndef isup_grade(result, bincount=None):\n    if bincount is None:\n        bincount = np.bincount(result[0].flatten())\n    if len(bincount) <= 3:\n        return 0\n\n    bincount[0], bincount[1], bincount[2] = 0, 0, 0\n    maximum = 0\n    first = 0\n    second = 0\n\n    for i, count in enumerate(bincount):\n        if count > maximum:\n            first = i\n            maximum = count\n\n    if not first:\n        return 0\n\n    maximum = 0\n    bincount[first] = 0\n    for i, count in enumerate(bincount):\n        if count > maximum:\n            second = i\n            maximum = count\n\n    if not second:\n        second = first\n    isup = {\n        '3+3': 1,\n        '3+4': 2,\n        '4+3': 3,\n        '4+4': 4,\n        '3+5': 4,\n        '5+3': 4,\n        '4+5': 4,\n        '5+4': 5,\n        '5+5': 5\n    }\n    return isup[f'{first}+{second}']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dblock, dls = create_datablock(bs=8)\ndls.show_batch(max_n=6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn = unet_learner(dls, resnet34)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.model = learn.model.to('cuda')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Approximately 15 seconds\nruntime('Create model and push to GPU')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# skip\n# results = learn.lr_find(start_lr=1e-7, end_lr=10, num_it=1000)\n# results","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Approximately 3.5 minutes\n# learning_rate = results.lr_min\nlearning_rate = 0.00010568174766376615","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"runtime('Determine learning rate')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS = 5\nlearn.fit(EPOCHS, learning_rate)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"runtime(f'Train {EPOCHS} Epochs')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nTEST_PATH = '../input/prostate-cancer-grade-assessment/test_images'\nTRAIN_PATH = '../input/prostate-cancer-grade-assessment/train_images'\n\n\ndef submit(path, fallback):\n    target = './submission.csv'\n    \n    submission = []\n\n    # Try to run on the real images, otherwise just use the test set\n    try:\n        images = os.listdir(path)\n    except FileNotFoundError:\n        path = fallback\n        images = os.listdir(path)[:1100]\n\n    # Predict for each file\n    for i in images:\n        file_path = os.path.join(path, i)\n        _id = i.replace('.tiff', '')\n    \n        with learn.no_bar():\n            grade = isup_grade(learn.predict(file_path))\n\n        submission.append((str(_id), str(grade)))\n\n    # Write out the data\n    with open('./submission.csv', 'wb') as f:\n        f.write(bytes('image_id,isup_grade\\n', encoding='utf8'))\n        for image in submission:\n            f.write(bytes(f'{image[0]},{image[1]}\\n', encoding='utf8'))\n                \nsubmit(TEST_PATH, TRAIN_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!cat ./submission.csv | head -n 10","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"runtime(f'Submit some results!')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}