{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -qU python-gdcm pydicom pylibjpeg\n!pip install polars\n!pip install lets-plot","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-08-07T03:14:13.515226Z","iopub.execute_input":"2023-08-07T03:14:13.516037Z","iopub.status.idle":"2023-08-07T03:15:01.056525Z","shell.execute_reply.started":"2023-08-07T03:14:13.515992Z","shell.execute_reply":"2023-08-07T03:15:01.055419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Imports\nimport os\nimport cv2\nimport glob\nimport gdcm\nimport torch\nimport random\nimport pydicom\nimport torchvision\nimport numpy as np\nimport polars as pl\nimport torch.nn as nn\nimport statistics as stats\nimport torch.optim as optim\nimport matplotlib.pyplot as plt\nimport torch.nn.functional as F\n\nfrom PIL import Image\nfrom lets_plot import *\nfrom torchvision import transforms\nfrom sklearn.metrics import confusion_matrix\nfrom torchvision.models import resnet50, ResNet50_Weights\nfrom sklearn.metrics import roc_auc_score, accuracy_score, precision_score, recall_score\nfrom lets_plot.mapping import as_discrete\nfrom torch.utils.data import Dataset, DataLoader\n\n# So the plots look nice\nLetsPlot.setup_html()\n","metadata":{"execution":{"iopub.status.busy":"2023-08-07T03:15:01.058317Z","iopub.execute_input":"2023-08-07T03:15:01.0587Z","iopub.status.idle":"2023-08-07T03:15:06.200958Z","shell.execute_reply.started":"2023-08-07T03:15:01.058668Z","shell.execute_reply":"2023-08-07T03:15:06.199592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pl.read_csv('/kaggle/input/rsna-breast-cancer-detection/train.csv')\ndf_test = pl.read_csv('/kaggle/input/rsna-breast-cancer-detection/test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-08-07T03:15:06.202648Z","iopub.execute_input":"2023-08-07T03:15:06.203492Z","iopub.status.idle":"2023-08-07T03:15:06.474228Z","shell.execute_reply.started":"2023-08-07T03:15:06.203447Z","shell.execute_reply":"2023-08-07T03:15:06.47334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.glimpse()","metadata":{"execution":{"iopub.status.busy":"2023-08-07T03:15:06.478159Z","iopub.execute_input":"2023-08-07T03:15:06.478623Z","iopub.status.idle":"2023-08-07T03:15:06.504224Z","shell.execute_reply.started":"2023-08-07T03:15:06.47858Z","shell.execute_reply":"2023-08-07T03:15:06.503414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Missing values by column')\ndf_train.select(pl.all().is_null().sum())","metadata":{"execution":{"iopub.status.busy":"2023-08-07T03:15:06.505824Z","iopub.execute_input":"2023-08-07T03:15:06.506514Z","iopub.status.idle":"2023-08-07T03:15:06.558004Z","shell.execute_reply.started":"2023-08-07T03:15:06.50648Z","shell.execute_reply":"2023-08-07T03:15:06.557246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target='cancer'\ncolor1='#d8e2dc'\ncolor2='#f4acb7'\ncolor3='#ee4266'\n\ndf_plt = df_train.with_columns(\n    pl.when(pl.col('cancer') == 1).then('Cancer Present').otherwise('No Cancer').alias('cancer'),\n    pl.when((pl.col('cancer') == 1) & (pl.col('invasive') == 1))\\\n        .then('Invasive')\\\n        .when((pl.col('cancer') == 1) & (pl.col('invasive') == 0))\\\n        .then('Non-Invasive')\\\n        .when((pl.col('cancer') == 0) & (pl.col('invasive') == 0))\\\n        .then('No Cancer')\\\n        .otherwise('No Cancer')\\\n        .alias('invasive2')\n)\n\n# Target variable plot\nvar = 'cancer'\ntitle = 'Cancer Distribution'\nlegend_title = ''\n\nplt1 = \\\n    ggplot(df_plt)+\\\n    geom_bar(aes(x = as_discrete(var),\n                fill = as_discrete(var)),\n            color = None,\n            size = 0.5)+\\\n    scale_fill_manual(values = [color1, color2])+\\\n    theme_minimal()+\\\n    theme(\n        plot_title = element_text(hjust = 0.5, face = 'bold'),\n        legend_position = \"top\",\n        panel_grid_major = element_blank(),\n        panel_grid_minor = element_blank(),\n        legend_title = element_blank(),\n        axis_title_x = element_blank(),\n        axis_line_y = element_line(size = 1))+\\\n    coord_flip()+\\\n    labs(y = \"Count\", title = title)\n\n# Target variable by invasiveness\nvar = 'invasive2'\ntitle = 'Cancer Distribution by Invasiveness'\nlegend_title = ''\n\nplt2 = \\\n    ggplot(df_plt)+\\\n    geom_bar(aes(x = as_discrete(var),\n                fill = as_discrete(var)),\n            color = None,\n            size = 0.5)+\\\n    scale_fill_manual(values = [color1, color3, color2])+\\\n    theme_minimal()+\\\n    theme(\n        plot_title = element_text(hjust = 0.5, face = 'bold'),\n        legend_position = \"top\",\n        panel_grid_major = element_blank(),\n        panel_grid_minor = element_blank(),\n        legend_title = element_blank(),\n        axis_title_x = element_blank(),\n        axis_line_y = element_line(size = 1))+\\\n    coord_flip()+\\\n    labs(y = \"Count\", title = title)\n\n\n\nbunch = GGBunch()\nbunch.add_plot(plt1, 0, 0, 500, 250)\nbunch.add_plot(plt2, 520, 0, 500, 250)\nbunch.show()","metadata":{"execution":{"iopub.status.busy":"2023-08-07T03:15:06.559424Z","iopub.execute_input":"2023-08-07T03:15:06.560019Z","iopub.status.idle":"2023-08-07T03:15:12.010305Z","shell.execute_reply.started":"2023-08-07T03:15:06.559988Z","shell.execute_reply":"2023-08-07T03:15:12.008891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# By cancer status\nvar = 'age'\ntitle = 'Age by Cancer Status'\nxlab = 'Age (years)'\n\nplt1 =\\\n    ggplot(df_plt)+\\\n    geom_density(aes(x = var, fill = target), \n                 color = 'gray', alpha = 0.5)+\\\n    scale_fill_manual(values = [color1, color2])+\\\n    theme(plot_title = element_text(hjust = 0.5, face = 'bold'),\n         legend_position = \"top\",\n         legend_title = element_blank())+\\\n    labs(x = xlab, y = 'Proportion', title = title)\n\nplt2 =\\\n    ggplot(df_plt)+\\\n    geom_boxplot(aes(x = target, y = var, \n                     fill = target), outlier_shape = 21)+\\\n    scale_fill_manual(values = [color1, color2])+\\\n    scale_x_continuous(labels=['     ','     '])+\\\n    coord_flip()+\\\n    theme_minimal()+\\\n    theme(plot_title = element_text(hjust = 0.5, face = 'bold'),\n          legend_position = \"top\",\n          legend_title = element_blank())+\\\n    labs(x = '', y = xlab, title = '')\n\n# By invasiveness\nvar = 'age'\ntitle = 'Age by Cancer Invasiveness'\nxlab = 'Age (years)'\n\nplt3 =\\\n    ggplot(df_plt)+\\\n    geom_density(aes(x = var, fill = 'invasive2'), \n                 color = 'gray', alpha = 0.5)+\\\n    scale_fill_manual(values = [color1, color3, color2])+\\\n    theme(plot_title = element_text(hjust = 0.5, face = 'bold'),\n         legend_position = \"top\",\n         legend_title = element_blank())+\\\n    labs(x = xlab, y = 'Proportion', title = title)\n\nplt4 =\\\n    ggplot(df_plt)+\\\n    geom_boxplot(aes(x = 'invasive2', y = var, \n                     fill = 'invasive2'), outlier_shape = 21)+\\\n    scale_fill_manual(values = [color1, color3, color2])+\\\n    scale_x_continuous(labels=['     ','     '])+\\\n    coord_flip()+\\\n    theme_minimal()+\\\n    theme(plot_title = element_text(hjust = 0.5, face = 'bold'),\n          legend_position = \"top\",\n          legend_title = element_blank())+\\\n    labs(x = '', y = xlab, title = '')\n\nbunch = GGBunch()\nbunch.add_plot(plt1, 0, 0, 500, 250)\nbunch.add_plot(plt2, 0, 250, 500, 250)\nbunch.add_plot(plt3, 520, 0, 500, 250)\nbunch.add_plot(plt4, 520, 250, 500, 250)\nbunch.show()","metadata":{"execution":{"iopub.status.busy":"2023-08-07T03:15:12.012285Z","iopub.execute_input":"2023-08-07T03:15:12.01325Z","iopub.status.idle":"2023-08-07T03:15:26.218656Z","shell.execute_reply.started":"2023-08-07T03:15:12.013211Z","shell.execute_reply":"2023-08-07T03:15:26.217401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function for getting a dicom plot from letsplot\ndef get_dicom_plt(dcm_path, title):\n    \n    dcm = pydicom.dcmread(dcm_path)\n    img = dcm.pixel_array\n    img = (img - img.min()) / (img.max() - img.min())\n    \n    if dcm.PhotometricInterpretation == \"MONOCHROME1\":\n        img = 1 - img\n        \n    plt=\\\n        ggplot()+\\\n        geom_imshow(img)+\\\n        theme(\n            legend_position='none', \n            panel_grid=element_blank(), \n            axis=element_blank(),\n            plot_title=element_text(hjust=0.5, face='bold'))+\\\n        labs(title=title)\n    \n    return plt","metadata":{"execution":{"iopub.status.busy":"2023-08-07T03:15:26.220119Z","iopub.execute_input":"2023-08-07T03:15:26.220492Z","iopub.status.idle":"2023-08-07T03:15:26.228726Z","shell.execute_reply.started":"2023-08-07T03:15:26.22046Z","shell.execute_reply":"2023-08-07T03:15:26.227232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Getting info based on cancer status\ninvasive_patients = df_plt.filter(pl.col('invasive2') == 'Invasive').select(['patient_id', 'image_id'])\ninvasive_patient_ids = invasive_patients.get_column('patient_id')\ninvasive_img_ids = invasive_patients.get_column('image_id')\n\nnoninvasive_patients = df_plt.filter(pl.col('invasive2') == 'Non-Invasive').select(['patient_id', 'image_id'])\nnoninvasive_patient_ids = noninvasive_patients.get_column('patient_id')\nnoninvasive_img_ids = noninvasive_patients.get_column('image_id')\n\nno_cancer_patients = df_plt.filter(pl.col('invasive2') == 'No Cancer').select(['patient_id', 'image_id'])\nno_cancer_patient_ids = no_cancer_patients.get_column('patient_id')\nno_cancer_img_ids = no_cancer_patients.get_column('image_id')\n\n# Initializing stuff\nimg_dir = '/kaggle/input/rsna-breast-cancer-detection/train_images'\nnpatients = 3\nbunch = GGBunch()\n\n# Plotting\nfor i in range(npatients):\n    \n    # For invasive cancer patients\n    dcm_path = f'{img_dir}/{invasive_patient_ids[i]}/{invasive_img_ids[i]}.dcm'\n    bunch.add_plot(get_dicom_plt(dcm_path, title = 'Invasive Cancer'), 0 + i*300, 0, 300, 300)\n    \n    # For noninvasive cancer patients\n    dcm_path = f'{img_dir}/{noninvasive_patient_ids[i]}/{noninvasive_img_ids[i]}.dcm'\n    bunch.add_plot(get_dicom_plt(dcm_path, title = 'Non-Invasive Cancer'), 0 + i*300, 310, 300, 300)\n\n    # For those without cancer\n    dcm_path = f'{img_dir}/{no_cancer_patient_ids[i]}/{no_cancer_img_ids[i]}.dcm'\n    bunch.add_plot(get_dicom_plt(dcm_path, title = 'No Cancer'), 0 + i*300, 620, 300, 300)\n                   \nbunch.show()","metadata":{"execution":{"iopub.status.busy":"2023-08-07T03:15:26.230216Z","iopub.execute_input":"2023-08-07T03:15:26.231623Z","iopub.status.idle":"2023-08-07T03:15:54.549793Z","shell.execute_reply.started":"2023-08-07T03:15:26.231578Z","shell.execute_reply":"2023-08-07T03:15:54.547433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Downsampling the non-cancerous targets and creating a final df\ndf_target1 = df_train.filter(pl.col('cancer') == 1)\ndf_target0 = df_train.filter(pl.col('cancer') == 0)\n\nn_idx0 = len(df_target0) # Number of noncancerous patients\n\ndf_target0 = df_target0\\\n    .with_row_count()\\\n    .filter(pl.col('row_nr').is_in(random.sample(range(n_idx0), len(df_target1))))\\\n    .drop('row_nr')\n\n# Final df\ndf_keep = \\\n    pl.concat([df_target1, df_target0], how='vertical')\\\n    .select(pl.all().shuffle(seed=19970507))\n\n# Imputing missing values for age, normalizing\nage_mean = round(df_keep.get_column('age').mean())\nage_min = df_keep.get_column('age').min()\nage_max = df_keep.get_column('age').max()\n\ndf_keep = \\\n    df_keep.with_columns(\n    \n        pl.when(pl.col('age') == None)\\\n        .then(age_mean)\\\n        .otherwise(pl.col('age'))\\\n        .alias('age')\n    \n    ).with_columns(\n    \n        ((pl.col('age') - age_min)/(age_max - age_min)).alias('age')\n    \n    ).to_dummies(\n    \n        ['view', 'laterality', 'implant']\n    \n    )\n\n# Adding a train/valid indicator column\nn = len(df_keep)\nntrain = round(n*.80)\n\ndf_keep =\\\n    df_keep.with_row_count()\\\n    .with_columns(\n        pl.when(pl.col('row_nr') <= ntrain)\\\n        .then('train')\\\n        .otherwise('valid')\\\n        .alias('trainvalid')\n    )\\\n    .drop('row_nr')\n    \n# Getting train/valid filenames and labels\n# Note the 256x256 pngs are names {patient_id}_{image_id}\ndf_keep = df_keep\\\n    .with_columns(pl.lit('_').alias('underscore'))\\\n    .with_columns(\n        pl.concat_str(\n            [\n                pl.col('patient_id'),\n                pl.col('underscore'),\n                pl.col('image_id')\n            ]\n        ).alias('fname')\n    ).drop('underscore')\n\ndf_train_meta = df_keep.filter(pl.col('trainvalid') == 'train')\ndf_valid_meta = df_keep.filter(pl.col('trainvalid') == 'valid')\n\ntrain_fnames = df_train_meta.get_column('fname')\nvalid_fnames = df_valid_meta.get_column('fname')\n\ntrain_labels = df_train_meta.get_column('cancer').to_numpy()\nvalid_labels = df_valid_meta.get_column('cancer').to_numpy()","metadata":{"execution":{"iopub.status.busy":"2023-08-07T03:15:54.552558Z","iopub.execute_input":"2023-08-07T03:15:54.552889Z","iopub.status.idle":"2023-08-07T03:15:54.655086Z","shell.execute_reply.started":"2023-08-07T03:15:54.552861Z","shell.execute_reply":"2023-08-07T03:15:54.654053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Training Metadata Characteristics')\ndf_train_meta.glimpse()","metadata":{"execution":{"iopub.status.busy":"2023-08-07T03:15:54.656633Z","iopub.execute_input":"2023-08-07T03:15:54.657253Z","iopub.status.idle":"2023-08-07T03:15:54.663792Z","shell.execute_reply.started":"2023-08-07T03:15:54.657216Z","shell.execute_reply":"2023-08-07T03:15:54.662871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Validation Metadata Characteristics')\ndf_valid_meta.glimpse()","metadata":{"execution":{"iopub.status.busy":"2023-08-07T03:55:03.033654Z","iopub.execute_input":"2023-08-07T03:55:03.034125Z","iopub.status.idle":"2023-08-07T03:55:03.04134Z","shell.execute_reply.started":"2023-08-07T03:55:03.03409Z","shell.execute_reply":"2023-08-07T03:55:03.040406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Validation Metadata Characteristics')\ndf_valid_meta.glimpse()","metadata":{"execution":{"iopub.status.busy":"2023-08-07T04:03:54.88429Z","iopub.execute_input":"2023-08-07T04:03:54.88478Z","iopub.status.idle":"2023-08-07T04:03:54.89172Z","shell.execute_reply.started":"2023-08-07T04:03:54.884746Z","shell.execute_reply":"2023-08-07T04:03:54.890588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get directories\nimg_dir='/kaggle/input/rsna-breast-cancer-256-pngs'\ntrain_dir='/kaggle/working/train'\nvalid_dir='/kaggle/working/valid'\n\n# Create directories for train/valid images\nos.makedirs(train_dir, exist_ok=True)\nos.makedirs(valid_dir, exist_ok=True)\n\n# Move the images into the correct directories\nfor file in train_fnames:\n    img = Image.open(f'{img_dir}/{file}.png')\n    img.save(f'{train_dir}/{file}.png')","metadata":{"execution":{"iopub.status.busy":"2023-08-07T04:04:27.221786Z","iopub.execute_input":"2023-08-07T04:04:27.222507Z","iopub.status.idle":"2023-08-07T04:04:27.291745Z","shell.execute_reply.started":"2023-08-07T04:04:27.222469Z","shell.execute_reply":"2023-08-07T04:04:27.290312Z"},"trusted":true},"execution_count":null,"outputs":[]}]}