{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Mammography Conv2d with Cropped Images","metadata":{}},{"cell_type":"markdown","source":"#### Outline of this notebook\n* a notebook for internet off environment\n* read various types of dicom\n* train images cropped images prepared in adnvance\n* test images cropped images created here\n* use image size of (224,224,3)\n* when cropping use image of (224,224)\n* when fitting use image of (224,224,3)\n* unbalanced data so use equivalent number of positive and negative\n* images are placed in kaggle/temp not in kaggle/working\n* submission resulted in Out of Memory error","metadata":{}},{"cell_type":"code","source":"!pip install pylibjpeg --no-index --find-links=file:///kaggle/input/read-dicom-set/dicom_read\n!pip install pylibjpeg-openjpeg --no-index --find-links=file:///kaggle/input/read-dicom-set/dicom_read\n!pip install pylibjpeg-libjpeg --no-index --find-links=file:///kaggle/input/read-dicom-set/dicom_read\n!pip install pydicom --no-index --find-links=file:///kaggle/input/read-dicom-set/dicom_read\n!pip install python-gdcm --no-index --find-links=file:///kaggle/input/read-dicom-set/dicom_read\n!pip install dicomsdl --no-index --find-links=file:///kaggle/input/read-dicom-set/dicom_read","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-01-01T13:12:41.301937Z","iopub.execute_input":"2023-01-01T13:12:41.302833Z","iopub.status.idle":"2023-01-01T13:13:49.652509Z","shell.execute_reply.started":"2023-01-01T13:12:41.302712Z","shell.execute_reply":"2023-01-01T13:13:49.65134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/competitions/rsna-breast-cancer-detection/discussion/370045\n# in case of internet-on environment\n# !pip install -U pylibjpeg pylibjpeg-openjpeg pylibjpeg-libjpeg pydicom python-gdcm","metadata":{"execution":{"iopub.status.busy":"2023-01-01T13:13:49.655934Z","iopub.execute_input":"2023-01-01T13:13:49.656441Z","iopub.status.idle":"2023-01-01T13:13:49.661708Z","shell.execute_reply.started":"2023-01-01T13:13:49.656393Z","shell.execute_reply":"2023-01-01T13:13:49.660408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nimport os\nimport cv2\nimport random\nimport shutil\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.preprocessing.image import load_img, img_to_array\nfrom tensorflow.python.keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.metrics import classification_report, log_loss, accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten\nfrom keras.callbacks import EarlyStopping\nfrom keras.layers.convolutional import Conv2D, MaxPooling2D\nfrom keras.utils import np_utils\nimport pylibjpeg\nfrom libjpeg import decode\nimport pydicom as dicom #### pydicom\nimport dicomsdl         #### dicomsdl\nfrom pydicom import dcmread\nfrom pydicom.data import get_testdata_file\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport skimage\nfrom skimage.transform import resize","metadata":{"papermill":{"duration":5.493148,"end_time":"2021-05-24T13:38:27.942967","exception":false,"start_time":"2021-05-24T13:38:22.449819","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-01T13:13:49.663128Z","iopub.execute_input":"2023-01-01T13:13:49.663456Z","iopub.status.idle":"2023-01-01T13:13:56.486919Z","shell.execute_reply.started":"2023-01-01T13:13:49.663426Z","shell.execute_reply":"2023-01-01T13:13:56.485811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir /kaggle/temp\n!mkdir /kaggle/temp/image\n!mkdir /kaggle/temp/image/train_images\n!mkdir /kaggle/temp/image/test_images","metadata":{"execution":{"iopub.status.busy":"2023-01-01T13:13:56.488348Z","iopub.execute_input":"2023-01-01T13:13:56.488934Z","iopub.status.idle":"2023-01-01T13:14:00.803413Z","shell.execute_reply.started":"2023-01-01T13:13:56.488902Z","shell.execute_reply":"2023-01-01T13:14:00.802272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# input processed & cropped train data","metadata":{}},{"cell_type":"code","source":"#copped train image\npaths1=[]\nfor dirname, _, filenames in os.walk('/kaggle/input/mammography-image-cropping-4-1a'):\n    for filename in filenames:\n        if filename[-4:]=='.png':\n            paths1+=[(os.path.join(dirname, filename))]","metadata":{"execution":{"iopub.status.busy":"2023-01-01T13:14:00.807021Z","iopub.execute_input":"2023-01-01T13:14:00.807432Z","iopub.status.idle":"2023-01-01T13:14:20.158768Z","shell.execute_reply.started":"2023-01-01T13:14:00.807395Z","shell.execute_reply":"2023-01-01T13:14:20.157929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#copped train image\npaths2=[]\nfor dirname, _, filenames in os.walk('/kaggle/input/mammography-image-cropping-4-2a'):\n    for filename in filenames:\n        if filename[-4:]=='.png':\n            paths2+=[(os.path.join(dirname, filename))]","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-01-01T13:14:20.160504Z","iopub.execute_input":"2023-01-01T13:14:20.160943Z","iopub.status.idle":"2023-01-01T13:14:36.526945Z","shell.execute_reply.started":"2023-01-01T13:14:20.1609Z","shell.execute_reply":"2023-01-01T13:14:36.52597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#copped train image to temp\nfor path in tqdm(paths1):\n    if path.split('/')[-2]=='train_images':\n        file=path.split('/')[-1]\n        shutil.copy(path,'/kaggle/temp/image/train_images/'+file)\nfor path in tqdm(paths2):\n    file=path.split('/')[-1]\n    if path.split('/')[-2]=='train_images':\n        shutil.copy(path,'/kaggle/temp/image/train_images/'+file)\n        \n    #elif path.split('/')[-2]=='test_images':\n       #shutil.copy(path,'/kaggle/working/image/test_images/'+file)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-01-01T13:14:36.528257Z","iopub.execute_input":"2023-01-01T13:14:36.528702Z","iopub.status.idle":"2023-01-01T13:19:30.563193Z","shell.execute_reply.started":"2023-01-01T13:14:36.528664Z","shell.execute_reply":"2023-01-01T13:19:30.562006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# process test dicom data ","metadata":{}},{"cell_type":"code","source":"paths=[]\nfiles=[]\nids=[]\nfor dirname, _, filenames in os.walk('/kaggle/input/rsna-breast-cancer-detection/test_images'):\n    for filename in filenames:\n        if filename[-4:]!='.csv':\n            paths+=[(os.path.join(dirname, filename))]\n            files+=[filename[:-4]]\n            ids+=[dirname.split('/')[-1]]","metadata":{"execution":{"iopub.status.busy":"2023-01-01T13:19:30.564693Z","iopub.execute_input":"2023-01-01T13:19:30.565837Z","iopub.status.idle":"2023-01-01T13:19:30.577217Z","shell.execute_reply.started":"2023-01-01T13:19:30.565792Z","shell.execute_reply":"2023-01-01T13:19:30.575977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test images ong to temp\ndir0='/kaggle/temp/image/test_images/'\ndataset=[]\nerrors=[]\nnewfiles=[]\n\nfor i,path in enumerate(paths):\n    img = dicomsdl.open(path)   ### usage for dicomsdl\n    pip=img.getPixelDataInfo()['PhotometricInterpretation']   ### usage for dicomsdl\n\n    try:\n        data=img.pixelData()   ### usage for dicomsdl\n        data=data-np.min(data)\n        #print(data.shape) ###\n        if np.max(data) != 0:\n            data=data/np.max(data)\n        data=(data*255).astype(np.uint8) \n        if pip=='MONOCHROME2':\n            data=255-data\n        #data=cv2.cvtColor(data,cv2.COLOR_BGR2RGB)\n        data=cv2.resize(data,dsize=(224,224))\n        #data=cv2.cvtColor(data,cv2.COLOR_BGR2GRAY)\n        #print(data.shape) ###32,32,3   \n        #plt.imshow(data)\n        #plt.show()\n        ps=path.split('/')[-3:]\n        newfile=ps[-2]+'_'+ps[-1][0:-4]+'.png'\n        print(newfile)\n        cv2.imwrite(os.path.join(dir0,newfile),data)\n        dataset+=[data]\n        newfiles+=[newfile]\n        \n    except:\n        errors+=[i]\n        newfiles+=['N']\n\nprint(len(dataset),len(errors))\n#DATA.to_csv('DATA.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-01T13:19:30.770041Z","iopub.execute_input":"2023-01-01T13:19:30.77052Z","iopub.status.idle":"2023-01-01T13:19:34.374496Z","shell.execute_reply.started":"2023-01-01T13:19:30.770453Z","shell.execute_reply":"2023-01-01T13:19:34.372349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/train.csv')\ntest=pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/test.csv')\nprint(train.columns.tolist())","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-01-01T13:19:34.37663Z","iopub.execute_input":"2023-01-01T13:19:34.379744Z","iopub.status.idle":"2023-01-01T13:19:34.602381Z","shell.execute_reply.started":"2023-01-01T13:19:34.379472Z","shell.execute_reply":"2023-01-01T13:19:34.601005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train2=train[['patient_id','image_id','laterality','cancer']]\ndisplay(train2)\ntest2=test[['patient_id','image_id','laterality']]\ntcancer=pd.Series(data=['0']*4,name='cancer')\ntest2=pd.concat([test2,tcancer],axis=1)\ndisplay(test2)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-01-01T13:19:34.603928Z","iopub.execute_input":"2023-01-01T13:19:34.604375Z","iopub.status.idle":"2023-01-01T13:19:34.644362Z","shell.execute_reply.started":"2023-01-01T13:19:34.604333Z","shell.execute_reply":"2023-01-01T13:19:34.643315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir='/kaggle/temp/image/train_images/'\ntest_dir='/kaggle/temp/image/test_images/'\ntrain2=train2.astype(str)\ntest2=test2.astype(str)\ntrain2['path']=train_dir+train2['patient_id']+'_'+train2['image_id']+'.png'\ntest2['path']=test_dir+test2['patient_id']+'_'+test2['image_id']+'.png'","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-01-01T13:19:34.645987Z","iopub.execute_input":"2023-01-01T13:19:34.64629Z","iopub.status.idle":"2023-01-01T13:19:34.827467Z","shell.execute_reply.started":"2023-01-01T13:19:34.646248Z","shell.execute_reply":"2023-01-01T13:19:34.826646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train2['cancer'].value_counts()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-01-01T13:19:36.026445Z","iopub.execute_input":"2023-01-01T13:19:36.027491Z","iopub.status.idle":"2023-01-01T13:19:36.193535Z","shell.execute_reply.started":"2023-01-01T13:19:36.027439Z","shell.execute_reply":"2023-01-01T13:19:36.192358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_posi=train2[train2['cancer']=='1'].reset_index(drop=True)\ntrain_nega=train2[train2['cancer']=='0'].reset_index(drop=True)\nn=len(train_nega)\nprint(n)\nN=list(range(n))\nrandom.seed(2023)\nrandom.shuffle(N)\ntrain_nega2=train_nega.iloc[N[0:(n//50)]]","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-01-01T13:19:36.194711Z","iopub.execute_input":"2023-01-01T13:19:36.195824Z","iopub.status.idle":"2023-01-01T13:19:36.309675Z","shell.execute_reply.started":"2023-01-01T13:19:36.195791Z","shell.execute_reply":"2023-01-01T13:19:36.308453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train3=pd.concat([train_posi,train_nega2],axis=0).reset_index(drop=True)\nprint(len(train3))\ndisplay(train3)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-01-01T13:19:36.311347Z","iopub.execute_input":"2023-01-01T13:19:36.311671Z","iopub.status.idle":"2023-01-01T13:19:36.33132Z","shell.execute_reply.started":"2023-01-01T13:19:36.311643Z","shell.execute_reply":"2023-01-01T13:19:36.33032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_img=[]\nfor i in tqdm(range(len(train3))):\n    path=train3.loc[i,'path']\n    img=cv2.imread(path)\n    #img=cv2.resize(np.array(img),dsize=(150,150))\n    #print(img.shape)\n    #img=cv2.cvtColor(img,cv2.COLOR_BGR2RGB)\n    #print(img.shape)\n    train_img+=[img]","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","papermill":{"duration":545.160614,"end_time":"2021-05-24T13:47:33.216089","exception":false,"start_time":"2021-05-24T13:38:28.055475","status":"completed"},"tags":[],"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-01-01T13:19:36.332987Z","iopub.execute_input":"2023-01-01T13:19:36.333425Z","iopub.status.idle":"2023-01-01T13:19:40.579392Z","shell.execute_reply.started":"2023-01-01T13:19:36.333385Z","shell.execute_reply":"2023-01-01T13:19:40.578236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_img=[]\nfor i in tqdm(range(len(test2))):\n    path=test2.loc[i,'path']\n    img=cv2.imread(path)\n    print(img.shape)\n    #img=cv2.cvtColor(img,cv2.COLOR_BGR2RGB)\n    #img=cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)\n    test_img+=[img[:,:,0]]\n    #plt.imshow(img)\n    #plt.show()","metadata":{"papermill":{"duration":102.934458,"end_time":"2021-05-24T13:49:16.752699","exception":false,"start_time":"2021-05-24T13:47:33.818241","status":"completed"},"tags":[],"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-01-01T13:19:40.599199Z","iopub.execute_input":"2023-01-01T13:19:40.599668Z","iopub.status.idle":"2023-01-01T13:19:40.684338Z","shell.execute_reply.started":"2023-01-01T13:19:40.599624Z","shell.execute_reply":"2023-01-01T13:19:40.683548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n# test image cropped start","metadata":{}},{"cell_type":"code","source":"#croping image1\ndef crop_image(array, image_size, noise):\n    threshold = 10\n    threshold2 = 150\n    # to avoid noise on the boundary\n\n    if np.mean(array, axis = 1)[0] > noise + 10:\n        array = array[2:,:]\n    if np.mean(array, axis = 1)[-1] > noise + 10:\n        array = array[0:-3,:]\n    if np.mean(array, axis = 0)[0] > noise + 10:\n        array = array[:,2:]\n    if np.mean(array, axis = 0)[-1] > noise + 10:\n        array = array[:,0:-3]\n    \n    x_max = np.quantile(array, axis = 1, q = 0.95)\n    x_max2 = np.where(x_max >= threshold)[0]     \n    \n    T = 0.02\n    while x_max2.shape[0] < 2:\n        x_max = np.quantile(array, axis = 1, q = 0.95 + T)\n        x_max2 = np.where(x_max >= threshold)[0] \n        T += 0.02\n        T = min(T, 0.05)\n    \n    y_max = np.quantile(array, axis = 0, q = 0.95)    \n    y_max2 = np.where(y_max >= threshold)[0] \n    \n    T = 0.02\n    while y_max2.shape[0] < 2:\n        y_max = np.quantile(array, axis = 0, q = 0.95 + T)    \n        y_max2 = np.where(y_max >= threshold)[0] \n        T += 0.02\n        T = min(T, 0.05)\n    \n    x_max_m = np.max(array, axis = 1)\n    y_max_m = np.max(array, axis = 0)    \n    x_max2_m = np.where(x_max_m >= threshold2)[0]   \n    y_max2_m = np.where(y_max_m >= threshold2)[0]         \n    \n    T = 10\n    while x_max2_m.shape[0] < 2:\n        x_max2_m = np.where(x_max_m >= threshold2 - T)[0] \n        T += 25\n    T = 10\n    while y_max2_m.shape[0] < 2:\n        y_max2_m = np.where(y_max_m >= threshold2 - T)[0] \n        T += 25    \n    \n    x1 = int(np.mean(np.array([x_max2[0], x_max2_m[0]])))\n    x2 = int(np.mean(np.array([x_max2[-1], x_max2_m[-1]]))) \n    \n    y1 = int(np.mean(np.array([y_max2[0], y_max2_m[0]])))\n    y2 = int(np.mean(np.array([y_max2[-1], y_max2_m[-1]])))\n    \n    margin = 3\n    LX = x2-x1\n    LY = y2-y1    \n    DL = LX - LY\n    ML = np.max([x2-x1, y2-y1])\n    array2 = np.zeros((ML + 2*margin, ML + 2*margin))\n    \n    # centering\n    if DL > 0:\n        array2[margin:-margin,int(abs(DL)/2)+margin:int(abs(DL)/2)+LY+margin] = array[x1:x2,y1:y2]\n    \n    else:\n        array2[int(abs(DL)/2)+margin:int(abs(DL)/2)+LX+margin,margin:-margin] = array[x1:x2,y1:y2]\n\n    return resize(array2, (image_size, image_size), preserve_range=True).astype(\"uint8\")","metadata":{"execution":{"iopub.status.busy":"2023-01-01T13:19:40.68577Z","iopub.execute_input":"2023-01-01T13:19:40.686583Z","iopub.status.idle":"2023-01-01T13:19:40.707812Z","shell.execute_reply.started":"2023-01-01T13:19:40.686551Z","shell.execute_reply":"2023-01-01T13:19:40.706508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_img=np.array(test_img)\nimage_size=(224,224)\nnoise=10\nprint(test_img.shape)\nn0=len(test_img)\nIM_SIZE=224","metadata":{"execution":{"iopub.status.busy":"2023-01-01T13:19:40.709705Z","iopub.execute_input":"2023-01-01T13:19:40.710585Z","iopub.status.idle":"2023-01-01T13:19:40.726193Z","shell.execute_reply.started":"2023-01-01T13:19:40.710525Z","shell.execute_reply":"2023-01-01T13:19:40.724529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#croping image2\ntest_m = np.zeros((n0, IM_SIZE, IM_SIZE), dtype = \"uint8\")\ntest_before = np.zeros((n0,224,224), dtype = \"uint8\")\ntest_m0 = np.zeros((224,224), dtype = \"int16\")\n    \nk = 0\nfor j in range(n0):\n    test_df0 = test_img\n    test_m0[:,:] = np.reshape(np.array(test_df0[j,:,:], dtype = \"int16\"),(224,224))\n    test_m0[:,:] = - test_m0[:,:] + 255\n    add = 255 - np.max(test_m0)\n    test_m0[:,:] = test_m0[:,:] + add\n    noise = 65 + add\n    test_m0[test_m0 < noise] = 0 # Noise Reduction\n    test_m[k,:,:] = crop_image(test_m0, image_size = IM_SIZE, noise = noise).astype(\"uint8\")\n    test_before[k,:,:] = test_m0\n    k += 1 ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#croping image3\ntest_before2=[]\ntest_m2=[]\nfor i in range(len(test_before)):\n    img=test_before[i]\n    #print(img.shape)\n    img=255-img\n    test_before2+=[img]\n    img2=test_m[i]\n    img2=255-img2\n\n    img2=cv2.cvtColor(img2, cv2.COLOR_BGR2RGB)\n    test_m2+=[img2]\n    \n    path=paths[i]\n    ps=path.split('/')[-3:]\n    cv2.imwrite(os.path.join(ps[-2],ps[-1][0:-4]+'.png'),img2)\n    \ntest_before2=np.array(test_before2)\ntest_m2=np.array(test_m2)\nprint(test_before2.shape,test_m2.shape)\n#np.save('./mammo1', test_m2)\ntest_img=test_m2","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"test image cropped completed\n\n---","metadata":{}},{"cell_type":"code","source":"trainX=np.array(train_img)\ntestX=np.array(test_img)\nprint(trainX.shape,testX.shape)","metadata":{"papermill":{"duration":0.85156,"end_time":"2021-05-24T13:49:20.788541","exception":false,"start_time":"2021-05-24T13:49:19.936981","status":"completed"},"tags":[],"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-01-01T13:19:41.127686Z","iopub.status.idle":"2023-01-01T13:19:41.129615Z","shell.execute_reply.started":"2023-01-01T13:19:41.129224Z","shell.execute_reply":"2023-01-01T13:19:41.129259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainY0=train3['cancer']\ntrainY=to_categorical(trainY0)","metadata":{"papermill":{"duration":0.739536,"end_time":"2021-05-24T13:49:22.260405","exception":false,"start_time":"2021-05-24T13:49:21.520869","status":"completed"},"tags":[],"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-01-01T13:19:41.131109Z","iopub.status.idle":"2023-01-01T13:19:41.131674Z","shell.execute_reply.started":"2023-01-01T13:19:41.1314Z","shell.execute_reply":"2023-01-01T13:19:41.131426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"trainx,testx,trainy,testy=train_test_split(trainX,trainY,test_size=0.2,random_state=44)","metadata":{"papermill":{"duration":0.790162,"end_time":"2021-05-24T13:49:23.776686","exception":false,"start_time":"2021-05-24T13:49:22.986524","status":"completed"},"tags":[],"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-01-01T13:19:41.133189Z","iopub.status.idle":"2023-01-01T13:19:41.133767Z","shell.execute_reply.started":"2023-01-01T13:19:41.133482Z","shell.execute_reply":"2023-01-01T13:19:41.13351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential()\nmodel.add(Conv2D(32,(4,4),input_shape = (224,224,3),activation = 'relu'))\nmodel.add(MaxPooling2D(pool_size=(2,2)))\nmodel.add(Conv2D(64,(3,3),activation = 'relu'))\nmodel.add(MaxPooling2D(pool_size=(2,2)))\nmodel.add(Dropout(0.2))\nmodel.add(Flatten())\nmodel.add(Dense(104, activation='relu'))\nmodel.add(Dense(52, activation='relu'))\nmodel.add(Dense(26, activation='softmax'))\nmodel.add(Dense(2, activation='softmax'))\nmodel.compile(loss='categorical_crossentropy', optimizer='adam',metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-01-01T13:19:41.135326Z","iopub.status.idle":"2023-01-01T13:19:41.135872Z","shell.execute_reply.started":"2023-01-01T13:19:41.135595Z","shell.execute_reply":"2023-01-01T13:19:41.135627Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"his=model.fit(trainx,trainy,batch_size=8,validation_data=(testx,testy),epochs=200)","metadata":{"execution":{"iopub.status.busy":"2023-01-01T13:19:41.137072Z","iopub.status.idle":"2023-01-01T13:19:41.137731Z","shell.execute_reply.started":"2023-01-01T13:19:41.137369Z","shell.execute_reply":"2023-01-01T13:19:41.137398Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_acc = his.history['accuracy']\nvalue_acc = his.history['val_accuracy']\nget_loss = his.history['loss']\nvalidation_loss = his.history['val_loss']\n\nepochs = range(len(get_acc))\nplt.plot(epochs, get_acc, 'r', label='Accuracy of Training data')\nplt.plot(epochs, value_acc, 'b', label='Accuracy of Validation data')\nplt.title('Training vs validation accuracy')\nplt.legend(loc=0)\nplt.figure()\nplt.show()\n\nepochs = range(len(get_loss))\nplt.plot(epochs, get_loss, 'r', label='Loss of Training data')\nplt.plot(epochs, validation_loss, 'b', label='Loss of Validation data')\nplt.title('Training vs validation loss')\nplt.legend(loc=0)\nplt.figure()\nplt.show()","metadata":{"papermill":{"duration":1.789493,"end_time":"2021-05-24T13:54:01.77904","exception":false,"start_time":"2021-05-24T13:53:59.989547","status":"completed"},"tags":[],"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-01-01T13:19:41.140016Z","iopub.status.idle":"2023-01-01T13:19:41.140704Z","shell.execute_reply.started":"2023-01-01T13:19:41.140421Z","shell.execute_reply":"2023-01-01T13:19:41.140448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict using test data","metadata":{}},{"cell_type":"code","source":"preds=model.predict(testX)\ndisplay(preds)\npreds2=[]\nfor pred in preds:\n    preds2+=[pred[0]]\nprint(preds2)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-01-01T13:19:41.145506Z","iopub.status.idle":"2023-01-01T13:19:41.147118Z","shell.execute_reply.started":"2023-01-01T13:19:41.146063Z","shell.execute_reply":"2023-01-01T13:19:41.146092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submit","metadata":{}},{"cell_type":"code","source":"test2['prediction_id']=test2['patient_id'].astype(str) + \"_\" + test2['laterality']\npred_ids=test2[['prediction_id']]\nPRED=pd.Series(data=preds2,name='cancer')\nPRED2=pd.concat([pred_ids,PRED],axis=1)\nPRED3=PRED2.groupby('prediction_id',as_index=False).max().reset_index(drop=True)\nPRED3.to_csv('submission.csv',index=False)\ndisplay(PRED3)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-01-01T13:19:41.14862Z","iopub.status.idle":"2023-01-01T13:19:41.150777Z","shell.execute_reply.started":"2023-01-01T13:19:41.150432Z","shell.execute_reply":"2023-01-01T13:19:41.150461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":1.390015,"end_time":"2021-05-24T13:54:20.243696","exception":false,"start_time":"2021-05-24T13:54:18.853681","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":1.580468,"end_time":"2021-05-24T13:54:23.192901","exception":false,"start_time":"2021-05-24T13:54:21.612433","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}