{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# import module","metadata":{}},{"cell_type":"code","source":"# pip install -qU python-gdcm pydicom pylibjpeg","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:21:58.198764Z","iopub.execute_input":"2023-01-15T03:21:58.199635Z","iopub.status.idle":"2023-01-15T03:21:58.207327Z","shell.execute_reply.started":"2023-01-15T03:21:58.199576Z","shell.execute_reply":"2023-01-15T03:21:58.205473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport cv2\nimport pandas as pd\nimport pydicom\nimport glob\n#import gdcm\n# import pylibjpeg\nimport scipy\nfrom skimage import data\nfrom skimage import filters\nfrom skimage.color import rgb2gray\nimport os, sys, tarfile\nimport matplotlib.pyplot as plt\nfrom torchvision.io import read_image\nfrom tqdm.notebook import tqdm\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom random import randrange\nfrom tqdm.notebook import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-15T03:21:58.210833Z","iopub.execute_input":"2023-01-15T03:21:58.211833Z","iopub.status.idle":"2023-01-15T03:21:58.222088Z","shell.execute_reply.started":"2023-01-15T03:21:58.211774Z","shell.execute_reply":"2023-01-15T03:21:58.220982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA meta data","metadata":{"execution":{"iopub.status.busy":"2022-12-31T08:16:52.706463Z","iopub.execute_input":"2022-12-31T08:16:52.707Z","iopub.status.idle":"2022-12-31T08:16:52.72775Z","shell.execute_reply.started":"2022-12-31T08:16:52.706881Z","shell.execute_reply":"2022-12-31T08:16:52.726553Z"}}},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/train.csv')\ntest_df = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/test.csv')\nprint(\"train files: \", len(train_df))\nprint(\"test files: \", len(test_df))\n\npd.reset_option('max_colwidth')","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:21:58.22347Z","iopub.execute_input":"2023-01-15T03:21:58.224325Z","iopub.status.idle":"2023-01-15T03:21:58.398141Z","shell.execute_reply.started":"2023-01-15T03:21:58.224281Z","shell.execute_reply":"2023-01-15T03:21:58.396596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:21:58.401739Z","iopub.execute_input":"2023-01-15T03:21:58.402154Z","iopub.status.idle":"2023-01-15T03:21:58.432825Z","shell.execute_reply.started":"2023-01-15T03:21:58.402118Z","shell.execute_reply":"2023-01-15T03:21:58.431574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:21:58.434498Z","iopub.execute_input":"2023-01-15T03:21:58.435781Z","iopub.status.idle":"2023-01-15T03:21:58.45302Z","shell.execute_reply.started":"2023-01-15T03:21:58.435728Z","shell.execute_reply":"2023-01-15T03:21:58.451852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x=train_df[\"cancer\"],palette='gnuplot').set(title='Cancer Distribution');","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:21:58.454503Z","iopub.execute_input":"2023-01-15T03:21:58.455695Z","iopub.status.idle":"2023-01-15T03:21:58.909165Z","shell.execute_reply.started":"2023-01-15T03:21:58.455637Z","shell.execute_reply":"2023-01-15T03:21:58.907891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['cancer'].value_counts(normalize=True).plot(kind='pie', autopct='%.2f');","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:21:58.910873Z","iopub.execute_input":"2023-01-15T03:21:58.911243Z","iopub.status.idle":"2023-01-15T03:21:59.063202Z","shell.execute_reply.started":"2023-01-15T03:21:58.911211Z","shell.execute_reply":"2023-01-15T03:21:59.061228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This Data imbalanced","metadata":{}},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:21:59.065354Z","iopub.execute_input":"2023-01-15T03:21:59.065881Z","iopub.status.idle":"2023-01-15T03:21:59.101247Z","shell.execute_reply.started":"2023-01-15T03:21:59.065818Z","shell.execute_reply":"2023-01-15T03:21:59.099378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"age\"] = pd.to_numeric(train_df[\"age\"])\n\nsorted_ages = np.sort(train_df[\"age\"].values)\nprint(sorted_ages)","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:21:59.109992Z","iopub.execute_input":"2023-01-15T03:21:59.111349Z","iopub.status.idle":"2023-01-15T03:21:59.127403Z","shell.execute_reply.started":"2023-01-15T03:21:59.111273Z","shell.execute_reply":"2023-01-15T03:21:59.125677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(train_df.corr(),cmap='gnuplot');","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:21:59.130101Z","iopub.execute_input":"2023-01-15T03:21:59.131275Z","iopub.status.idle":"2023-01-15T03:21:59.580952Z","shell.execute_reply.started":"2023-01-15T03:21:59.131173Z","shell.execute_reply":"2023-01-15T03:21:59.579573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Best way to extract cancer is biospy","metadata":{}},{"cell_type":"code","source":"sns.histplot(data=train_df['age'],color='purple',bins=30,binwidth=3)\nplt.xlim(0, 100);","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:21:59.582685Z","iopub.execute_input":"2023-01-15T03:21:59.583175Z","iopub.status.idle":"2023-01-15T03:21:59.902734Z","shell.execute_reply.started":"2023-01-15T03:21:59.583126Z","shell.execute_reply":"2023-01-15T03:21:59.901459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Age of Patient is from 40 to 80","metadata":{}},{"cell_type":"code","source":"sns.histplot(data=train_df,x='age',hue='cancer',bins=40);","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:21:59.904068Z","iopub.execute_input":"2023-01-15T03:21:59.905854Z","iopub.status.idle":"2023-01-15T03:22:00.409089Z","shell.execute_reply.started":"2023-01-15T03:21:59.905812Z","shell.execute_reply":"2023-01-15T03:22:00.407677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Cancer patient age is from 40 to 75 ","metadata":{}},{"cell_type":"code","source":"sns.countplot(x=train_df['laterality']);","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:22:00.411065Z","iopub.execute_input":"2023-01-15T03:22:00.411603Z","iopub.status.idle":"2023-01-15T03:22:00.633912Z","shell.execute_reply.started":"2023-01-15T03:22:00.411552Z","shell.execute_reply":"2023-01-15T03:22:00.63293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x=train_df['view']);","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:22:00.63538Z","iopub.execute_input":"2023-01-15T03:22:00.635997Z","iopub.status.idle":"2023-01-15T03:22:00.884459Z","shell.execute_reply.started":"2023-01-15T03:22:00.63596Z","shell.execute_reply":"2023-01-15T03:22:00.883413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x=train_df['difficult_negative_case']);","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:22:00.885939Z","iopub.execute_input":"2023-01-15T03:22:00.886529Z","iopub.status.idle":"2023-01-15T03:22:01.085381Z","shell.execute_reply.started":"2023-01-15T03:22:00.886475Z","shell.execute_reply":"2023-01-15T03:22:01.084144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['BIRADS'].unique()","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:22:01.087129Z","iopub.execute_input":"2023-01-15T03:22:01.088379Z","iopub.status.idle":"2023-01-15T03:22:01.096467Z","shell.execute_reply.started":"2023-01-15T03:22:01.088337Z","shell.execute_reply":"2023-01-15T03:22:01.095429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x=train_df['BIRADS']);","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:22:01.09783Z","iopub.execute_input":"2023-01-15T03:22:01.098586Z","iopub.status.idle":"2023-01-15T03:22:01.324071Z","shell.execute_reply.started":"2023-01-15T03:22:01.098529Z","shell.execute_reply":"2023-01-15T03:22:01.322875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Crop_Image","metadata":{}},{"cell_type":"code","source":"def get_masks_and_sizes_of_connected_components(img_mask):\n    \"\"\"\n    Finds the connected components from the mask of the image\n    \"\"\"\n    mask, num_labels = scipy.ndimage.label(img_mask)\n\n    mask_pixels_dict = {}\n    for i in range(num_labels+1):\n        this_mask = (mask == i)\n        if img_mask[this_mask][0] != 0:\n            # Exclude the 0-valued mask\n            mask_pixels_dict[i] = np.sum(this_mask)\n        \n    return mask, mask_pixels_dict\n\n\ndef get_mask_of_largest_connected_component(img_mask):\n    \"\"\"\n    Finds the largest connected component from the mask of the image\n    \"\"\"\n    mask, mask_pixels_dict = get_masks_and_sizes_of_connected_components(img_mask)\n    largest_mask_index = pd.Series(mask_pixels_dict).idxmax()\n    largest_mask = mask == largest_mask_index\n    return largest_mask","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:22:01.325683Z","iopub.execute_input":"2023-01-15T03:22:01.327051Z","iopub.status.idle":"2023-01-15T03:22:01.336429Z","shell.execute_reply.started":"2023-01-15T03:22:01.327011Z","shell.execute_reply":"2023-01-15T03:22:01.334494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def super_procescing(img):\n    clahe_img = np.copy(img)\n    threshold = filters.threshold_isodata(clahe_img)\n    clahe = cv2.createCLAHE(clipLimit =threshold)\n    clahe_img = clahe.apply(clahe_img) -(threshold+3)\n    bin_img = (clahe_img > threshold)\n    kernel = np.ones((5, 5), np.uint8)\n    bin_img = bin_img.astype('uint8')\n    bin_img = cv2.erode(bin_img, kernel)\n    zeros=np.zeros(bin_img.shape)\n    bin_img=(bin_img==zeros)\n    img_mask = get_mask_of_largest_connected_component(bin_img)\n    #crop_image\n\n    farest_pixel = np.max(list(zip(*np.where(img_mask == 1))), axis=0)\n    nearest_pixel = np.min(list(zip(*np.where(img_mask == 1))), axis=0)\n    croped =  img[nearest_pixel[0]:farest_pixel[0], nearest_pixel[1]:farest_pixel[1]]\n    return(croped)","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:22:01.338286Z","iopub.execute_input":"2023-01-15T03:22:01.338835Z","iopub.status.idle":"2023-01-15T03:22:01.356995Z","shell.execute_reply.started":"2023-01-15T03:22:01.338789Z","shell.execute_reply":"2023-01-15T03:22:01.355315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"str(train_df.iloc[0].patient_id)+'_'+str(train_df.iloc[0].image_id)","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:22:01.358772Z","iopub.execute_input":"2023-01-15T03:22:01.359275Z","iopub.status.idle":"2023-01-15T03:22:01.37311Z","shell.execute_reply.started":"2023-01-15T03:22:01.359226Z","shell.execute_reply":"2023-01-15T03:22:01.37204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape[0]","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:22:01.374269Z","iopub.execute_input":"2023-01-15T03:22:01.374656Z","iopub.status.idle":"2023-01-15T03:22:01.388379Z","shell.execute_reply.started":"2023-01-15T03:22:01.374623Z","shell.execute_reply":"2023-01-15T03:22:01.386824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Image in index 53065 is buged so we remove it from data_frame**","metadata":{}},{"cell_type":"code","source":"train_df=train_df.drop(53065)\ntrain_df.shape[0]","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:22:01.390708Z","iopub.execute_input":"2023-01-15T03:22:01.391514Z","iopub.status.idle":"2023-01-15T03:22:01.416441Z","shell.execute_reply.started":"2023-01-15T03:22:01.391443Z","shell.execute_reply":"2023-01-15T03:22:01.415038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.iloc[53065]","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:22:01.418446Z","iopub.execute_input":"2023-01-15T03:22:01.419255Z","iopub.status.idle":"2023-01-15T03:22:01.429864Z","shell.execute_reply.started":"2023-01-15T03:22:01.419205Z","shell.execute_reply":"2023-01-15T03:22:01.42845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[train_df['patient_id']==10130]","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:22:01.436343Z","iopub.execute_input":"2023-01-15T03:22:01.437377Z","iopub.status.idle":"2023-01-15T03:22:01.4608Z","shell.execute_reply.started":"2023-01-15T03:22:01.437331Z","shell.execute_reply":"2023-01-15T03:22:01.459518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_path='/kaggle/input/rsna-mammography-images-as-pngs/images_as_pngs_cv2_vl_512/train_images_processed_cv2_vl_512'\ncancer_list=[]\nfor i in tqdm(range(train_df.shape[0])):\n    if train_df.iloc[i].cancer == 1:\n        path=os.path.join(train_path,str(train_df.iloc[i].patient_id),str(train_df.iloc[i].image_id)+ '.png')\n        cancer_list.append(path)\nlen(cancer_list)","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:22:01.462087Z","iopub.execute_input":"2023-01-15T03:22:01.462436Z","iopub.status.idle":"2023-01-15T03:22:09.246787Z","shell.execute_reply.started":"2023-01-15T03:22:01.462405Z","shell.execute_reply":"2023-01-15T03:22:09.245547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#rand_indices = [randrange(len(train_df)) for x in range(0,10)]\ntrain_path='/kaggle/input/rsna-mammography-images-as-pngs/images_as_pngs_cv2_vl_512/train_images_processed_cv2_vl_512'\nn_cols = 10\nn_rows = 10\nfig, ax = plt.subplots(n_rows, n_cols, figsize=(n_cols*5,n_rows*5))\nfor r in tqdm(range(0,n_rows)):\n    for c in range(0,n_cols):\n        idx = r*n_cols + c #index loop through list \n        ax_idx = ax[r,c]\n        path = os.path.join(train_path,str(train_df.iloc[idx].patient_id),str(train_df.iloc[idx].image_id)+ '.png')\n        r_img = cv2.imread(path,0)\n        img= super_procescing(r_img)\n        ax_idx.imshow(img,cmap=plt.cm.gray)\n        ax_idx.axis('off')\n        ax_idx.set_title(str(train_df.iloc[idx].patient_id)+'_'+str(train_df.iloc[idx].image_id))\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-15T03:22:09.24831Z","iopub.execute_input":"2023-01-15T03:22:09.248658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir RSNA_Crop\n!mkdir RSNA_Crop/cancer\n!mkdir RSNA_Crop/nocancer\n!mkdir RSNA_Crop_CLAHE\n!mkdir RSNA_Crop_CLAHE/cancer\n!mkdir RSNA_Crop_CLAHE/nocancer","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_path='/kaggle/input/rsna-mammography-images-as-pngs/images_as_pngs_cv2_vl_512/train_images_processed_cv2_vl_512'\nlist_path=[]\nfor i in tqdm(range(len(train_df['image_id']))):\n    path = train_path+f'/{str(train_df.iloc[i].patient_id)}/{str(train_df.iloc[i].image_id)}.png'\n    list_path.append(path)\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[train_df.patient_id==822]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_croped(path):\n    patient=path.split('/')[-2]\n    image_id=path.split('/')[-1].split('.')[0]\n    if path in cancer_list:\n        img=cv2.imread(path,0)\n        image=super_procescing(img)\n        cv2.imwrite('/kaggle/working/RSNA_Crop/cancer'+f'/{patient}_{image_id}.png',image)\n    else:\n        img=cv2.imread(path,0)\n        image=super_procescing(img)\n        cv2.imwrite('/kaggle/working/RSNA_Crop/nocancer'+f'/{patient}_{image_id}.png',image)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import concurrent.futures as cf\n\nwith cf.ThreadPoolExecutor(max_workers=2) as executor:\n    saver = list(tqdm(executor.map(save_croped, list_path),total=len(list_path)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Crop CLAHE all Image","metadata":{"execution":{"iopub.status.busy":"2023-01-07T16:25:38.449937Z","iopub.execute_input":"2023-01-07T16:25:38.450371Z","iopub.status.idle":"2023-01-07T16:25:38.461829Z","shell.execute_reply.started":"2023-01-07T16:25:38.450339Z","shell.execute_reply":"2023-01-07T16:25:38.460569Z"}}},{"cell_type":"code","source":"def save_croped_clahe(path):\n    patient=path.split('/')[-2]\n    image_id=path.split('/')[-1].split('.')[0]\n    if path in cancer_list:\n        img=cv2.imread(path,0)\n        image=super_procescing(img)\n        clahe = cv2.createCLAHE(clipLimit = 20)\n        image = clahe.apply(image) + 30\n        cv2.imwrite('/kaggle/working/RSNA_Crop_CLAHE/cancer'+f'/{patient}_{image_id}.png',image)\n    else:\n        img=cv2.imread(path,0)\n        image=super_procescing(img)\n        clahe = cv2.createCLAHE(clipLimit = 20)\n        image = clahe.apply(image) + 30\n        cv2.imwrite('/kaggle/working/RSNA_Crop_CLAHE/nocancer'+f'/{patient}_{image_id}.png',image)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import concurrent.futures as cf\nwith cf.ThreadPoolExecutor(max_workers=2) as executor:\n    saved = list(tqdm(executor.map(save_croped_clahe, list_path),total=len(list_path)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}