{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Setup and Imports:","metadata":{}},{"cell_type":"code","source":"from fastai.imports import *\nfrom fastai.vision.all import *\nfrom fastai.metrics import BalancedAccuracy\nimport shutil\nimport os\n\n# Normal max size causes decompression bomb warnings\nImage.MAX_IMAGE_PIXELS = 10000000000","metadata":{"execution":{"iopub.status.busy":"2023-10-26T00:55:14.046697Z","iopub.execute_input":"2023-10-26T00:55:14.047554Z","iopub.status.idle":"2023-10-26T00:55:14.05271Z","shell.execute_reply.started":"2023-10-26T00:55:14.04752Z","shell.execute_reply":"2023-10-26T00:55:14.051723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = Path('../input/UBC-OCEAN')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-26T00:55:14.061548Z","iopub.execute_input":"2023-10-26T00:55:14.061896Z","iopub.status.idle":"2023-10-26T00:55:14.06665Z","shell.execute_reply.started":"2023-10-26T00:55:14.061859Z","shell.execute_reply":"2023-10-26T00:55:14.065816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Part 1: Data","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(path/\"train.csv\")\ntest_df = pd.read_csv(path/\"test.csv\")\nsubmission = pd.read_csv(path/\"sample_submission.csv\")\n\ndf.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-10-26T01:30:54.017949Z","iopub.execute_input":"2023-10-26T01:30:54.018352Z","iopub.status.idle":"2023-10-26T01:30:54.038212Z","shell.execute_reply.started":"2023-10-26T01:30:54.018324Z","shell.execute_reply":"2023-10-26T01:30:54.037369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"im = Image.open(f\"{path}/train_thumbnails/4_thumbnail.png\")\nim.to_thumb(1000,1000)","metadata":{"execution":{"iopub.status.busy":"2023-10-26T00:55:14.105405Z","iopub.execute_input":"2023-10-26T00:55:14.105675Z","iopub.status.idle":"2023-10-26T00:55:14.627778Z","shell.execute_reply.started":"2023-10-26T00:55:14.105652Z","shell.execute_reply":"2023-10-26T00:55:14.62651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def return_label(o):\n    return df.loc[df['image_id'] == int(\"\".join(filter(str.isdigit, Path(o).name)))][\"label\"].iloc[0]","metadata":{"execution":{"iopub.status.busy":"2023-10-26T00:55:14.629549Z","iopub.execute_input":"2023-10-26T00:55:14.629886Z","iopub.status.idle":"2023-10-26T00:55:14.634977Z","shell.execute_reply.started":"2023-10-26T00:55:14.629857Z","shell.execute_reply":"2023-10-26T00:55:14.634075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"return_label(\"/kaggle/input/UBC-OCEAN/train_images/4134.png\")","metadata":{"execution":{"iopub.status.busy":"2023-10-26T00:55:14.636104Z","iopub.execute_input":"2023-10-26T00:55:14.636375Z","iopub.status.idle":"2023-10-26T00:55:14.649016Z","shell.execute_reply.started":"2023-10-26T00:55:14.636353Z","shell.execute_reply":"2023-10-26T00:55:14.648206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_file_path(o, test=False):\n    base_path = f\"{path}/{'test' if test else 'train'}\" \n    if os.path.exists(f\"{base_path}_thumbnails/{o}_thumbnail.png\"):\n        return f\"{base_path}_thumbnails/{o}_thumbnail.png\"\n    elif os.path.exists(f\"{base_path}_images/{o}.png\"):\n        return f\"{base_path}_images/{o}.png\"\n    else:\n        raise OSError(o, \"Image does not exist in data\")","metadata":{"execution":{"iopub.status.busy":"2023-10-26T01:28:12.210824Z","iopub.execute_input":"2023-10-26T01:28:12.211511Z","iopub.status.idle":"2023-10-26T01:28:12.216927Z","shell.execute_reply.started":"2023-10-26T01:28:12.211468Z","shell.execute_reply":"2023-10-26T01:28:12.216028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_file_path(4)","metadata":{"execution":{"iopub.status.busy":"2023-10-26T01:46:07.40771Z","iopub.execute_input":"2023-10-26T01:46:07.408629Z","iopub.status.idle":"2023-10-26T01:46:07.415896Z","shell.execute_reply.started":"2023-10-26T01:46:07.408593Z","shell.execute_reply":"2023-10-26T01:46:07.415027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_cancer_images(path, test=False):\n    files = []\n    if test:\n        for i in test_df[\"image_id\"]:\n            files.append(get_file_path(i, test=True))\n    else:\n        for i in df[\"image_id\"]:\n            files.append(get_file_path(i))\n    return files","metadata":{"execution":{"iopub.status.busy":"2023-10-26T01:37:25.416992Z","iopub.execute_input":"2023-10-26T01:37:25.417815Z","iopub.status.idle":"2023-10-26T01:37:25.42315Z","shell.execute_reply.started":"2023-10-26T01:37:25.41778Z","shell.execute_reply":"2023-10-26T01:37:25.422089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_cancer_images(path, test=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-26T01:37:27.694769Z","iopub.execute_input":"2023-10-26T01:37:27.695659Z","iopub.status.idle":"2023-10-26T01:37:27.70408Z","shell.execute_reply.started":"2023-10-26T01:37:27.695625Z","shell.execute_reply":"2023-10-26T01:37:27.703177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls = DataBlock(\n    blocks=(ImageBlock, CategoryBlock),\n    get_items=get_cancer_images,\n    splitter=RandomSplitter(valid_pct=0.2, seed=42),\n    get_y=return_label,\n    item_tfms=[Resize(200, method=\"squish\")]\n).dataloaders(path)","metadata":{"execution":{"iopub.status.busy":"2023-10-26T01:37:37.03624Z","iopub.execute_input":"2023-10-26T01:37:37.036911Z","iopub.status.idle":"2023-10-26T01:37:41.035114Z","shell.execute_reply.started":"2023-10-26T01:37:37.036879Z","shell.execute_reply":"2023-10-26T01:37:41.034294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.train.show_batch(max_n=6)","metadata":{"execution":{"iopub.status.busy":"2023-10-26T01:37:43.177596Z","iopub.execute_input":"2023-10-26T01:37:43.177966Z","iopub.status.idle":"2023-10-26T01:38:04.759585Z","shell.execute_reply.started":"2023-10-26T01:37:43.177926Z","shell.execute_reply":"2023-10-26T01:38:04.758629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Part 2: Model","metadata":{}},{"cell_type":"code","source":"# Define metric function\nbalanced_accuracy = BalancedAccuracy()\n\nmodel = vision_learner(dls, resnet50, metrics=[accuracy, balanced_accuracy], path=\".\", pretrained=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-26T01:38:55.446367Z","iopub.execute_input":"2023-10-26T01:38:55.447294Z","iopub.status.idle":"2023-10-26T01:38:55.963561Z","shell.execute_reply.started":"2023-10-26T01:38:55.44725Z","shell.execute_reply":"2023-10-26T01:38:55.962769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fine_tune(3, 0.01)","metadata":{"execution":{"iopub.status.busy":"2023-10-26T01:38:58.477089Z","iopub.execute_input":"2023-10-26T01:38:58.478224Z","iopub.status.idle":"2023-10-26T01:43:47.87554Z","shell.execute_reply.started":"2023-10-26T01:38:58.478185Z","shell.execute_reply":"2023-10-26T01:43:47.874545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Part 3: Submission","metadata":{}},{"cell_type":"markdown","source":"Because of internet restrictions on this notebook, I'll be exporting the model to a separate notebook and doing inference on it there. [Linked here!](https://www.kaggle.com/japancolorado/inference-notebook/edit)","metadata":{}},{"cell_type":"code","source":"model.export()","metadata":{"execution":{"iopub.status.busy":"2023-10-26T01:44:07.019915Z","iopub.execute_input":"2023-10-26T01:44:07.020779Z","iopub.status.idle":"2023-10-26T01:44:07.29529Z","shell.execute_reply.started":"2023-10-26T01:44:07.020745Z","shell.execute_reply":"2023-10-26T01:44:07.294408Z"},"trusted":true},"execution_count":null,"outputs":[]}]}