{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#pip install efficientnet --quiet","metadata":{"execution":{"iopub.status.busy":"2021-09-17T22:23:57.725213Z","iopub.execute_input":"2021-09-17T22:23:57.725694Z","iopub.status.idle":"2021-09-17T22:23:57.73381Z","shell.execute_reply.started":"2021-09-17T22:23:57.725478Z","shell.execute_reply":"2021-09-17T22:23:57.732928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install git+https://github.com/qubvel/classification_models.git","metadata":{"execution":{"iopub.status.busy":"2021-09-17T22:23:57.73601Z","iopub.execute_input":"2021-09-17T22:23:57.73688Z","iopub.status.idle":"2021-09-17T22:24:07.208201Z","shell.execute_reply.started":"2021-09-17T22:23:57.736816Z","shell.execute_reply":"2021-09-17T22:24:07.207236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pydicom\nimport os\nimport matplotlib.pyplot as plt\nimport collections\nfrom tqdm import tqdm_notebook as tqdm\nfrom datetime import datetime\n\nfrom math import ceil, floor\nimport cv2\n\nimport tensorflow as tf\nimport keras\n\nfrom albumentations import (\n    Compose,\n    HorizontalFlip, ShiftScaleRotate,VerticalFlip,\n    RandomBrightness,RandomCrop,RandomContrast\n)\nimport sys\n#import cupy as cp\nfrom classification_models.keras import Classifiers\n\n#import albumentations\n# from keras_applications.resnet import ResNet50\n#from keras_applications.inception_v3 import InceptionV3\n#from efficientnet.keras import EfficientNetB2\n#from keras_applications.inception_resnet_v2 import InceptionResNetV2\n#from keras_applications.densenet import DenseNet121\nfrom sklearn.model_selection import ShuffleSplit\n\n#print(os.listdir('../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection'))\n\ntest_images_dir = '../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_test/'\ntrain_images_dir = '../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train/'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-17T22:24:07.209864Z","iopub.execute_input":"2021-09-17T22:24:07.210182Z","iopub.status.idle":"2021-09-17T22:24:10.679258Z","shell.execute_reply.started":"2021-09-17T22:24:07.210134Z","shell.execute_reply":"2021-09-17T22:24:10.678414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(os.listdir('../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train'))","metadata":{"execution":{"iopub.status.busy":"2021-09-17T22:24:10.680821Z","iopub.execute_input":"2021-09-17T22:24:10.681157Z","iopub.status.idle":"2021-09-17T22:24:23.400025Z","shell.execute_reply.started":"2021-09-17T22:24:10.681105Z","shell.execute_reply":"2021-09-17T22:24:23.399351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir('../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train')[0]","metadata":{"execution":{"iopub.status.busy":"2021-09-17T22:24:23.402581Z","iopub.execute_input":"2021-09-17T22:24:23.402874Z","iopub.status.idle":"2021-09-17T22:24:24.387612Z","shell.execute_reply.started":"2021-09-17T22:24:23.402828Z","shell.execute_reply":"2021-09-17T22:24:24.386596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 0. Sigmoid (brain + subudral + bone)\nMany thanks to [Ryan Epp](https://www.kaggle.com/reppic/gradient-sigmoid-windowing). Code is taken from his kernel (see his kernel for more information and other peoples work --- for example [David Tang](https://www.kaggle.com/dcstang/see-like-a-radiologist-with-systematic-windowing), [Marco](https://www.kaggle.com/marcovasquez/basic-eda-data-visualization), [Nanashi](https://www.kaggle.com/jesucristo/rsna-introduction-eda-models), and [Richard McKinley](https://www.kaggle.com/omission/eda-view-dicom-images-with-correct-windowing)). At first I thought I couldn't use sigmoid windowing for this kernel because of how expensive it is to do, but I could resize the image prior to the transformation to save a lot of computation. Not sure how much this will affect the performance of the training, but it really speeded it up.","metadata":{}},{"cell_type":"code","source":"from math import log\n\ndef correct_dcm(dcm):\n    x = dcm.pixel_array + 1000\n    px_mode = 4096\n    x[x>=px_mode] = x[x>=px_mode] - px_mode\n    dcm.PixelData = x.tobytes()\n    dcm.RescaleIntercept = -1000\n    return dcm\n\ndef window_image(dcm,window_center, window_width,desired_size,U=1.0, eps=(1.0 / 255.0)):\n    \n    if (dcm.BitsStored == 12) and (dcm.PixelRepresentation == 0) and (int(dcm.RescaleIntercept) > -100):\n        dcm = correct_dcm(dcm)\n    \n    img = dcm.pixel_array * dcm.RescaleSlope + dcm.RescaleIntercept\n    img = cv2.resize(img, desired_size[:2], interpolation=cv2.INTER_LINEAR)\n    \n    #img = cp.array(np.array(img))\n    img_min = window_center - window_width // 2\n    img_max = window_center + window_width // 2\n    img = np.clip(img, img_min, img_max)\n\n    return img\n\ndef bsb_window(dcm,desired_size=(256,256,3)):\n    brain_img = window_image(dcm, 40, 80,desired_size)\n    subdural_img = window_image(dcm, 80, 200,desired_size)\n    soft_img = window_image(dcm, 40, 380,desired_size)\n    \n    brain_img = (brain_img - 0) / 80\n    subdural_img = (subdural_img - (-20)) / 200\n    soft_img = (soft_img - (-150)) / 380\n    bsb_img = np.array([brain_img, subdural_img, soft_img]).transpose(1,2,0)\n\n    return bsb_img\n\n# Sanity Check\n# Example dicoms: ID_2669954a7, ID_5c8b5d701, ID_52c9913b1\n\ndicom = pydicom.dcmread(train_images_dir + 'ID_5c8b5d701' + '.dcm')\n#                                     ID  Label\n# 4045566          ID_5c8b5d701_epidural      0\n# 4045567  ID_5c8b5d701_intraparenchymal      1\n# 4045568  ID_5c8b5d701_intraventricular      0\n# 4045569      ID_5c8b5d701_subarachnoid      1\n# 4045570          ID_5c8b5d701_subdural      1\n# 4045571               ID_5c8b5d701_any      1\nplt.imshow(bsb_window(dicom), cmap=plt.cm.bone);\n","metadata":{"execution":{"iopub.status.busy":"2021-09-17T22:24:24.389167Z","iopub.execute_input":"2021-09-17T22:24:24.389594Z","iopub.status.idle":"2021-09-17T22:24:24.600382Z","shell.execute_reply.started":"2021-09-17T22:24:24.389415Z","shell.execute_reply":"2021-09-17T22:24:24.599645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check (with an example) if the correction works (visually)","metadata":{}},{"cell_type":"code","source":"def window_with_correction(dcm, window_center, window_width):\n    if (dcm.BitsStored == 12) and (dcm.PixelRepresentation == 0) and (int(dcm.RescaleIntercept) > -100):\n        dcm = correct_dcm(dcm)\n    img = dcm.pixel_array * dcm.RescaleSlope + dcm.RescaleIntercept\n    img_min = window_center - window_width // 2\n    img_max = window_center + window_width // 2\n    img = np.clip(img, img_min, img_max)\n    return img\n\ndef window_without_correction(dcm, window_center, window_width):\n    img = dcm.pixel_array * dcm.RescaleSlope + dcm.RescaleIntercept\n    img_min = window_center - window_width // 2\n    img_max = window_center + window_width // 2\n    img = np.clip(img, img_min, img_max)\n    return img\n\ndef window_testing(img, window):\n    brain_img = window(img, 40, 80)\n    subdural_img = window(img, 80, 200)\n    soft_img = window(img, 40, 380)\n    \n    brain_img = (brain_img - 0) / 80\n    subdural_img = (subdural_img - (-20)) / 200\n    soft_img = (soft_img - (-150)) / 380\n    bsb_img = np.array([brain_img, subdural_img, soft_img]).transpose(1,2,0)\n\n    return bsb_img\n\n# example of a \"bad data point\" (i.e. (dcm.BitsStored == 12) and (dcm.PixelRepresentation == 0) and (int(dcm.RescaleIntercept) > -100) == True)\ndicom = pydicom.dcmread(train_images_dir + \"ID_5c8b5d701\" + \".dcm\")\n\nfig, ax = plt.subplots(1, 2)\n\nax[0].imshow(window_testing(dicom, window_without_correction), cmap=plt.cm.bone);\nax[0].set_title(\"original\")\nax[1].imshow(window_testing(dicom, window_with_correction), cmap=plt.cm.bone);\nax[1].set_title(\"corrected\");","metadata":{"execution":{"iopub.status.busy":"2021-09-17T22:24:24.601978Z","iopub.execute_input":"2021-09-17T22:24:24.602335Z","iopub.status.idle":"2021-09-17T22:24:24.913782Z","shell.execute_reply.started":"2021-09-17T22:24:24.602281Z","shell.execute_reply":"2021-09-17T22:24:24.913129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1. Helper functions\n\n* read and transform dcms to 3-channel inputs for e.g. InceptionV3. \n* uses `sigmoid_bsb_window` from previous cell\n\n\\* Source for windowing (although now partly removed from this kernel): https://www.kaggle.com/omission/eda-view-dicom-images-with-correct-windowing","metadata":{}},{"cell_type":"code","source":"def _read(path, desired_size):\n    \"\"\"Will be used in DataGenerator\"\"\"\n    \n    dcm = pydicom.dcmread(path)\n    \n    try:\n        img = bsb_window(dcm,desired_size)\n    except:\n        img = np.zeros(desired_size)\n    \n    return img\n\n# Another sanity check \nplt.imshow(\n    _read(train_images_dir+'ID_5c8b5d701'+'.dcm', (256, 256)), cmap=plt.cm.bone\n);","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2021-09-17T22:24:24.91525Z","iopub.execute_input":"2021-09-17T22:24:24.915536Z","iopub.status.idle":"2021-09-17T22:24:25.086127Z","shell.execute_reply.started":"2021-09-17T22:24:24.915478Z","shell.execute_reply":"2021-09-17T22:24:25.085064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2. Data generators\n\nInherits from keras.utils.Sequence object and thus should be safe for multiprocessing.\n","metadata":{}},{"cell_type":"code","source":"class DataGenerator(keras.utils.Sequence):\n\n    def __init__(self, list_IDs, labels=None, batch_size=1, img_size=(512, 512, 1), \n                 img_dir=train_images_dir,augment=None, *args, **kwargs):\n\n        self.list_IDs = list_IDs\n        self.labels = labels\n        self.batch_size = batch_size\n        self.img_size = img_size\n        self.img_dir = img_dir\n        self.on_epoch_end()\n        self.augment = augment\n\n    def __len__(self):\n        return int(ceil(len(self.indices) / self.batch_size))\n\n    def __getitem__(self, index):\n        indices = self.indices[index*self.batch_size:(index+1)*self.batch_size]\n        list_IDs_temp = [self.list_IDs[k] for k in indices]\n        \n        if self.labels is not None:\n            X, Y = self.__data_generation(list_IDs_temp)\n            return X, Y\n        else:\n            X = self.__data_generation(list_IDs_temp)\n            return X\n        \n    def on_epoch_end(self):\n        \n        \n        if self.labels is not None: # for training phase we undersample and shuffle\n            # keep probability of any=0 and any=1\n            keep_prob = self.labels.iloc[:, 0].map({0: 0.35, 1: 0.5})\n            keep = (keep_prob > np.random.rand(len(keep_prob)))\n            self.indices = np.arange(len(self.list_IDs))[keep]\n            np.random.shuffle(self.indices)\n        else:\n            self.indices = np.arange(len(self.list_IDs))\n\n    def __data_generation(self, list_IDs_temp):\n        X = np.empty((self.batch_size, *self.img_size))\n        \n        if self.labels is not None: # training phase\n            Y = np.empty((self.batch_size, 6), dtype=np.float32)\n        \n            for i, ID in enumerate(list_IDs_temp):\n                X[i,] = _read(self.img_dir+ID+\".dcm\", self.img_size)\n                Y[i,] = self.labels.loc[ID].values\n        \n            if self.augment:\n                X = self.__augment(X)\n            return X, Y\n        \n        else: # test phase\n            for i, ID in enumerate(list_IDs_temp):\n                X[i,] = _read(self.img_dir+ID+\".dcm\", self.img_size)\n            \n            return X\n        \n    def __random_transform(self,img):\n        composition = Compose([\n            HorizontalFlip(),\n            VerticalFlip(),\n            ShiftScaleRotate(rotate_limit=45, shift_limit=0.15, scale_limit=0.15),\n            RandomBrightness(),\n            RandomContrast()\n        ])\n        \n        composed = composition(image=img)\n        aug_img = composed['image']\n        \n        return aug_img\n        \n    def __augment(self,img_batch):\n        for i in range(img_batch.shape[0]):\n            img_batch[i, ] = self.__random_transform(img_batch[i, ])\n        \n        return img_batch","metadata":{"execution":{"iopub.status.busy":"2021-09-17T22:24:25.087728Z","iopub.execute_input":"2021-09-17T22:24:25.088228Z","iopub.status.idle":"2021-09-17T22:24:25.112052Z","shell.execute_reply.started":"2021-09-17T22:24:25.088008Z","shell.execute_reply":"2021-09-17T22:24:25.110972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3a. loss function and metric","metadata":{}},{"cell_type":"code","source":"from keras import backend as K\n\ndef weighted_log_loss(y_true, y_pred):\n    \"\"\"\n    Can be used as the loss function in model.compile()\n    ---------------------------------------------------\n    \"\"\"\n    \n    class_weights = np.array([2., 1., 1., 1., 1., 1.])\n    \n    eps = K.epsilon()\n    \n    y_pred = K.clip(y_pred, eps, 1.0-eps)\n\n    out = -(         y_true  * K.log(      y_pred) * class_weights\n            + (1.0 - y_true) * K.log(1.0 - y_pred) * class_weights)\n    \n    return K.mean(out, axis=-1)\n\n\ndef _normalized_weighted_average(arr, weights=None):\n    \"\"\"\n    A simple Keras implementation that mimics that of \n    numpy.average(), specifically for this competition\n    \"\"\"\n    \n    if weights is not None:\n        scl = K.sum(weights)\n        weights = K.expand_dims(weights, axis=1)\n        return K.sum(K.dot(arr, weights), axis=1) / scl\n    return K.mean(arr, axis=1)\n\n\ndef weighted_loss(y_true, y_pred):\n    \"\"\"\n    Will be used as the metric in model.compile()\n    ---------------------------------------------\n    \n    Similar to the custom loss function 'weighted_log_loss()' above\n    but with normalized weights, which should be very similar \n    to the official competition metric:\n        https://www.kaggle.com/kambarakun/lb-probe-weights-n-of-positives-scoring\n    and hence:\n        sklearn.metrics.log_loss with sample weights\n    \"\"\"\n    \n    class_weights = K.variable([2., 1., 1., 1., 1., 1.])\n    \n    eps = K.epsilon()\n    \n    y_pred = K.clip(y_pred, eps, 1.0-eps)\n\n    loss = -(        y_true  * K.log(      y_pred)\n            + (1.0 - y_true) * K.log(1.0 - y_pred))\n    \n    loss_samples = _normalized_weighted_average(loss, class_weights)\n    \n    return K.mean(loss_samples)\n\n\ndef weighted_log_loss_metric(trues, preds):\n    \"\"\"\n    Will be used to calculate the log loss \n    of the validation set in PredictionCheckpoint()\n    ------------------------------------------\n    \"\"\"\n    class_weights = [2., 1., 1., 1., 1., 1.]\n    \n    epsilon = 1e-7\n    \n    preds = np.clip(preds, epsilon, 1-epsilon)\n    loss = trues * np.log(preds) + (1 - trues) * np.log(1 - preds)\n    loss_samples = np.average(loss, axis=1, weights=class_weights)\n\n    return - loss_samples.mean()\n\n","metadata":{"execution":{"iopub.status.busy":"2021-09-04T14:32:57.571902Z","iopub.execute_input":"2021-09-04T14:32:57.572222Z","iopub.status.idle":"2021-09-04T14:32:57.587674Z","shell.execute_reply.started":"2021-09-04T14:32:57.572167Z","shell.execute_reply":"2021-09-04T14:32:57.586862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3b. Model\n\nModel is divided into three parts: <br> \n\n* (REMOVED) The initial layer, which will transform/map input image of shape (\\_, \\_, 1) to another \"image\" of shape (\\_, \\_, 3).\n\n* The new input image is then passed through InceptionV3 (which I named \"engine\"). InceptionV3 could be replaced by any of the available architectures in keras_application.\n\n* Finally, the output from InceptionV3 goes through average pooling followed by two dense layers (including output layer).","metadata":{}},{"cell_type":"code","source":"\nclass PredictionCheckpoint(keras.callbacks.Callback):\n    \n    def __init__(self, test_df, valid_df, \n                 test_images_dir=test_images_dir, \n                 valid_images_dir=train_images_dir, \n                 batch_size=32, input_size=(224, 224, 3)):\n        \n        self.test_df = test_df\n        self.valid_df = valid_df\n        self.test_images_dir = test_images_dir\n        self.valid_images_dir = valid_images_dir\n        self.batch_size = batch_size\n        self.input_size = input_size\n        \n    def on_train_begin(self, logs={}):\n        self.test_predictions = []\n        self.valid_predictions = []\n        \n    def on_epoch_end(self,batch, logs={}):\n        self.test_predictions.append(\n            self.model.predict_generator(\n                DataGenerator(self.test_df.index, None, self.batch_size, self.input_size, self.test_images_dir), verbose=2)[:len(self.test_df)])\n        \n        # Commented out to save time\n#         self.valid_predictions.append(\n#             self.model.predict_generator(\n#                 DataGenerator(self.valid_df.index, None, self.batch_size, self.input_size, self.valid_images_dir), verbose=2)[:len(self.valid_df)])\n        \n#         print(\"validation loss: %.4f\" %\n#               weighted_log_loss_metric(self.valid_df.values, \n#                                    np.average(self.valid_predictions, axis=0, \n#                                               weights=[2**i for i in range(len(self.valid_predictions))])))\n        \n        # here you could also save the predictions with np.save()\n\n\nclass MyDeepModel:\n    \n    def __init__(self, engine, input_dims, batch_size=5, num_epochs=4, learning_rate=1e-3, \n                 decay_rate=1.0, decay_steps=1, weights=\"imagenet\", verbose=1):\n        \n        self.engine = engine\n        self.input_dims = input_dims\n        self.batch_size = batch_size\n        self.num_epochs = num_epochs\n        self.learning_rate = learning_rate\n        self.decay_rate = decay_rate\n        self.decay_steps = decay_steps\n        self.weights = weights\n        self.verbose = verbose\n        self._build()\n\n    def _build(self):\n        \n        \n        engine = self.engine(include_top=False, weights=self.weights, input_shape=self.input_dims,\n                             backend = keras.backend, layers = keras.layers,\n                             models = keras.models, utils = keras.utils)\n        \n        x = keras.layers.GlobalAveragePooling2D(name='avg_pool')(engine.output)\n        #x = keras.layers.Dropout(0.3)(x)\n#         x = keras.layers.Dense(keras.backend.int_shape(x)[1], activation=\"relu\", name=\"dense_hidden_1\")(x)\n#         x = keras.layers.Dropout(0.1)(x)\n        out = keras.layers.Dense(6, activation=\"sigmoid\", name='dense_output')(x)\n\n        self.model = keras.models.Model(inputs=engine.input, outputs=out)\n\n        self.model.compile(loss='binary_crossentropy', optimizer=keras.optimizers.Nadam(), metrics=[weighted_loss])\n    \n\n    def fit_and_predict(self, train_df, valid_df, test_df):\n        \n        # callbacks\n        pred_history = PredictionCheckpoint(test_df, valid_df, input_size=self.input_dims)\n        checkpoint = keras.callbacks.ModelCheckpoint('model.h5', monitor='weighted_loss',verbose=1, save_best_only=True)\n        scheduler = keras.callbacks.LearningRateScheduler(lambda epoch: self.learning_rate * pow(self.decay_rate, floor(epoch / self.decay_steps)))\n        \n        self.model.fit_generator(\n            DataGenerator(\n                train_df.index, \n                train_df, \n                self.batch_size, \n                self.input_dims, \n                train_images_dir,\n                augment=None\n            ),\n            epochs=self.num_epochs,\n            verbose=self.verbose,\n            use_multiprocessing=True,\n            workers=4,\n            callbacks=[checkpoint,scheduler]\n        )\n        \n        return pred_history\n    \n    def save(self, path):\n        self.model.save_weights(path)\n    \n    def load(self, path):\n        self.model.load_weights(path)","metadata":{"execution":{"iopub.status.busy":"2021-09-17T22:24:31.006205Z","iopub.execute_input":"2021-09-17T22:24:31.006513Z","iopub.status.idle":"2021-09-17T22:24:31.027306Z","shell.execute_reply.started":"2021-09-17T22:24:31.006459Z","shell.execute_reply":"2021-09-17T22:24:31.02651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4. Read csv files\n","metadata":{}},{"cell_type":"code","source":"def read_testset(filename=\"../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_sample_submission.csv\"):\n    df = pd.read_csv(filename)\n    df[\"Image\"] = df[\"ID\"].str.slice(stop=12)\n    df[\"Diagnosis\"] = df[\"ID\"].str.slice(start=13)\n    \n    df = df.loc[:, [\"Label\", \"Diagnosis\", \"Image\"]]\n    df = df.set_index(['Image', 'Diagnosis']).unstack(level=-1)\n    \n    return df\n\ndef read_trainset(filename=\"../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train.csv\"):\n    df = pd.read_csv(filename)\n    df[\"Image\"] = df[\"ID\"].str.slice(stop=12)\n    df[\"Diagnosis\"] = df[\"ID\"].str.slice(start=13)\n    \n    \"\"\"\n    print(df.shape)\n    ids = df['ID']\n    \n    duplicates = df[ids.isin(ids[ids.duplicated()])]\n    \n    print(duplicates['ID'].unique().shape,duplicates['ID'].shape)\n    \n    print(df[df.duplicated()==False].shape)\n    \n    duplicates_to_remove = [\n        1598538, 1598539, 1598540, 1598541, 1598542, 1598543,\n        312468,  312469,  312470,  312471,  312472,  312473,\n        2708700, 2708701, 2708702, 2708703, 2708704, 2708705,\n        3032994, 3032995, 3032996, 3032997, 3032998, 3032999\n    ]\n    \n    df = df.drop(index=duplicates_to_remove)\n    \"\"\"\n    df = df[df.duplicated()==False]\n    df = df.reset_index(drop=True)\n    \n    df = df.loc[:, [\"Label\", \"Diagnosis\", \"Image\"]]\n    df = df.set_index(['Image', 'Diagnosis']).unstack(level=-1)\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2021-09-17T22:24:36.153531Z","iopub.execute_input":"2021-09-17T22:24:36.154096Z","iopub.status.idle":"2021-09-17T22:24:36.164574Z","shell.execute_reply.started":"2021-09-17T22:24:36.154035Z","shell.execute_reply":"2021-09-17T22:24:36.163697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dupp = ['489ae4179', '854fba667', '921490062', 'a64d5deed']","metadata":{"execution":{"iopub.status.busy":"2021-09-17T22:24:36.849757Z","iopub.execute_input":"2021-09-17T22:24:36.850069Z","iopub.status.idle":"2021-09-17T22:24:36.853812Z","shell.execute_reply.started":"2021-09-17T22:24:36.850009Z","shell.execute_reply":"2021-09-17T22:24:36.853047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filename=\"../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train.csv\"\ndf = pd.read_csv(filename)\n\ndf[\"Image\"] = df[\"ID\"].str.slice(stop=12)\ndf[\"Diagnosis\"] = df[\"ID\"].str.slice(start=13)\ndf = df[df.duplicated()==False]\n\nidxx = df[\"Image\"].values\nll = int(len(df)/6)\nfor i in range(ll):\n    for j in range(6):\n        idxx[(i*6)+j] = idxx[(i*6)+j]+'_'+str(i)\n\n        \ndf[\"Image\"] = idxx\ndf","metadata":{"execution":{"iopub.status.busy":"2021-09-17T22:24:41.184797Z","iopub.execute_input":"2021-09-17T22:24:41.185148Z","iopub.status.idle":"2021-09-17T22:24:59.608829Z","shell.execute_reply.started":"2021-09-17T22:24:41.185089Z","shell.execute_reply":"2021-09-17T22:24:59.608177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df[df[\"Image\"]=='ID_'+ dupp[3]]","metadata":{"execution":{"iopub.status.busy":"2021-07-06T17:29:31.850909Z","iopub.execute_input":"2021-07-06T17:29:31.851513Z","iopub.status.idle":"2021-07-06T17:29:31.855767Z","shell.execute_reply.started":"2021-07-06T17:29:31.851431Z","shell.execute_reply":"2021-07-06T17:29:31.854685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df[df.duplicated()==False]\ndf = df.reset_index(drop=True)\n\ndf = df.loc[:, [\"Label\", \"Diagnosis\", \"Image\"]]\ndf","metadata":{"execution":{"iopub.status.busy":"2021-09-17T22:24:59.610653Z","iopub.execute_input":"2021-09-17T22:24:59.610948Z","iopub.status.idle":"2021-09-17T22:25:04.842521Z","shell.execute_reply.started":"2021-09-17T22:24:59.610891Z","shell.execute_reply":"2021-09-17T22:25:04.84171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.set_index(['Image', 'Diagnosis']).unstack(level=-1)\ndf","metadata":{"execution":{"iopub.status.busy":"2021-09-17T22:25:04.843984Z","iopub.execute_input":"2021-09-17T22:25:04.844297Z","iopub.status.idle":"2021-09-17T22:25:11.143914Z","shell.execute_reply.started":"2021-09-17T22:25:04.844251Z","shell.execute_reply":"2021-09-17T22:25:11.143116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_idx = np.array(df.index)\nlll = len(final_idx)\n\nfor i in range(lll):\n    final_idx[i] = final_idx[i][13:]","metadata":{"execution":{"iopub.status.busy":"2021-09-17T22:25:11.145416Z","iopub.execute_input":"2021-09-17T22:25:11.145865Z","iopub.status.idle":"2021-09-17T22:25:11.614031Z","shell.execute_reply.started":"2021-09-17T22:25:11.145687Z","shell.execute_reply":"2021-09-17T22:25:11.613059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_idx = np.array(final_idx,int)\nfinal_idx","metadata":{"execution":{"iopub.status.busy":"2021-09-17T22:25:11.616018Z","iopub.execute_input":"2021-09-17T22:25:11.616462Z","iopub.status.idle":"2021-09-17T22:25:11.736365Z","shell.execute_reply.started":"2021-09-17T22:25:11.616267Z","shell.execute_reply":"2021-09-17T22:25:11.735513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = read_testset()\ndf = read_trainset()","metadata":{"execution":{"iopub.status.busy":"2021-09-04T14:34:00.801454Z","iopub.execute_input":"2021-09-04T14:34:00.801796Z","iopub.status.idle":"2021-09-04T14:34:23.489954Z","shell.execute_reply.started":"2021-09-04T14:34:00.801736Z","shell.execute_reply":"2021-09-04T14:34:23.489119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-09-04T14:34:23.49185Z","iopub.execute_input":"2021-09-04T14:34:23.49232Z","iopub.status.idle":"2021-09-04T14:34:23.651824Z","shell.execute_reply.started":"2021-09-04T14:34:23.492124Z","shell.execute_reply":"2021-09-04T14:34:23.650577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-09-04T14:34:23.653837Z","iopub.execute_input":"2021-09-04T14:34:23.654563Z","iopub.status.idle":"2021-09-04T14:34:23.674658Z","shell.execute_reply.started":"2021-09-04T14:34:23.654344Z","shell.execute_reply":"2021-09-04T14:34:23.674014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 5. Train model and predict\n\n*Using train, validation and test set* <br>\n\nTraining for 4 epochs with Adam optimizer, with a learning rate of 0.0005 and decay rate of 0.8. The validation predictions are \\[exponentially weighted\\] averaged over all 4 epochs (same goes for the test set submission later). `fit_and_predict` returns validation and test predictions for all epochs.\n","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train set (00%) and validation set (10%)\nss = ShuffleSplit(n_splits=10, test_size=0.1, random_state=42).split(df.index)\n\n# lets go for the first fold only\ntrain_idx, valid_idx = next(ss)\n\n# obtain model\n# SE_resnext50, preprocess_input = Classifiers.get('seresnext50')\n\n# model = MyDeepModel(engine=SE_resnext50, input_dims=(224, 224, 3), batch_size=32, learning_rate=5e-4,\n#                     num_epochs=4, decay_rate=0.8, decay_steps=1, weights=\"imagenet\", verbose=1)\n\n","metadata":{"execution":{"iopub.status.busy":"2021-09-04T14:34:42.970832Z","iopub.execute_input":"2021-09-04T14:34:42.971149Z","iopub.status.idle":"2021-09-04T14:34:42.996531Z","shell.execute_reply.started":"2021-09-04T14:34:42.971091Z","shell.execute_reply":"2021-09-04T14:34:42.995797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# obtain test + validation predictions (history.test_predictions, history.valid_predictions)\n# history = model.fit_and_predict(df.iloc[train_idx], df.iloc[valid_idx], test_df)\n\n# model.model.load_weights('../input/inceptionv3model/model.h5')","metadata":{"execution":{"iopub.status.busy":"2021-07-06T17:30:20.641327Z","iopub.execute_input":"2021-07-06T17:30:20.641627Z","iopub.status.idle":"2021-07-06T17:30:20.645549Z","shell.execute_reply.started":"2021-07-06T17:30:20.641579Z","shell.execute_reply":"2021-07-06T17:30:20.644642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 6. Submit test predictions","metadata":{}},{"cell_type":"code","source":"# prediction = model.model.predict_generator(DataGenerator(test_df.index, None, 32, (224, 224, 3), test_images_dir), verbose=1)\n# print(prediction.shape)","metadata":{"execution":{"iopub.status.busy":"2021-07-06T17:30:20.64714Z","iopub.execute_input":"2021-07-06T17:30:20.647438Z","iopub.status.idle":"2021-07-06T17:30:20.660914Z","shell.execute_reply.started":"2021-07-06T17:30:20.647383Z","shell.execute_reply":"2021-07-06T17:30:20.659829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col = ['any','epidural','intraparenchymal','intraventricular','subarachnoid','subdural']","metadata":{"execution":{"iopub.status.busy":"2021-09-04T14:34:47.505109Z","iopub.execute_input":"2021-09-04T14:34:47.505427Z","iopub.status.idle":"2021-09-04T14:34:47.512906Z","shell.execute_reply.started":"2021-09-04T14:34:47.505375Z","shell.execute_reply":"2021-09-04T14:34:47.511782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df0 = df.iloc[train_idx]\nval_df0 = df.iloc[valid_idx]\ntrain_df0.shape,val_df0.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-04T14:34:49.94575Z","iopub.execute_input":"2021-09-04T14:34:49.946147Z","iopub.status.idle":"2021-09-04T14:34:50.178508Z","shell.execute_reply.started":"2021-09-04T14:34:49.946094Z","shell.execute_reply":"2021-09-04T14:34:50.177764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(6):\n    print(col[i],(train_df0['Label'][col[i]]==1).sum())\n    \n\nfor i in range(6):\n    print(col[i],(val_df0['Label'][col[i]]==1).sum())","metadata":{"execution":{"iopub.status.busy":"2021-09-04T14:34:55.149215Z","iopub.execute_input":"2021-09-04T14:34:55.149526Z","iopub.status.idle":"2021-09-04T14:34:55.200454Z","shell.execute_reply.started":"2021-09-04T14:34:55.149473Z","shell.execute_reply":"2021-09-04T14:34:55.199763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for train and valid\n\n# test_df0 = val_df0.iloc[:]\n# prediction = model.model.predict_generator(DataGenerator(test_df0.index, None, 32, (224, 224, 3), train_images_dir), verbose=1)\n# print(prediction.shape)\n# np.savez_compressed(f'pred_val.npz',data = prediction)\n# test_df0.iloc[:, :] = prediction[:len(test_df0)]\n# test_df0 = test_df0.stack().reset_index()\n# test_df0.insert(loc=0, column='ID', value=test_df0['Image'].astype(str) + \"_\" + test_df0['Diagnosis'])\n# test_df0 = test_df0.drop([\"Image\", \"Diagnosis\"], axis=1)\n# test_df0.to_csv('submission_val.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-06T17:30:21.170617Z","iopub.execute_input":"2021-07-06T17:30:21.17105Z","iopub.status.idle":"2021-07-06T17:30:21.175773Z","shell.execute_reply.started":"2021-07-06T17:30:21.170978Z","shell.execute_reply":"2021-07-06T17:30:21.17469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for train \n\n# test_df0 = train_df0.iloc[:]\n# prediction = model.model.predict_generator(DataGenerator(test_df0.index, None, 32, (224, 224, 3), train_images_dir), verbose=1)\n# print(prediction.shape)\n# np.savez_compressed(f'pred_train.npz',data = prediction)\n# test_df0.iloc[:, :] = prediction[:len(test_df0)]\n# test_df0 = test_df0.stack().reset_index()\n# test_df0.insert(loc=0, column='ID', value=test_df0['Image'].astype(str) + \"_\" + test_df0['Diagnosis'])\n# test_df0 = test_df0.drop([\"Image\", \"Diagnosis\"], axis=1)\n# test_df0.to_csv('submission_train.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-06T17:30:21.177171Z","iopub.execute_input":"2021-07-06T17:30:21.177437Z","iopub.status.idle":"2021-07-06T17:30:21.193344Z","shell.execute_reply.started":"2021-07-06T17:30:21.17739Z","shell.execute_reply":"2021-07-06T17:30:21.192427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df0 = test_df.iloc[:512]\n\n# prediction = model.model.predict_generator(DataGenerator(test_df0.index, None, 32, (224, 224, 3), test_images_dir), verbose=1)\n# print(prediction.shape)\n\n# np.savez_compressed(f'pred_test.npz',data = prediction)\n# test_df0.iloc[:, :] = prediction[:len(test_df0)]\n# test_df0 = test_df0.stack().reset_index()\n# test_df0.insert(loc=0, column='ID', value=test_df0['Image'].astype(str) + \"_\" + test_df0['Diagnosis'])\n# test_df0 = test_df0.drop([\"Image\", \"Diagnosis\"], axis=1)\n# test_df0.to_csv('submission_test.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-06T17:30:21.195098Z","iopub.execute_input":"2021-07-06T17:30:21.1955Z","iopub.status.idle":"2021-07-06T17:30:21.20391Z","shell.execute_reply.started":"2021-07-06T17:30:21.195417Z","shell.execute_reply":"2021-07-06T17:30:21.202867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# \"\"\"\n# try:\n#     test_df.iloc[:, :] = np.average(history.test_predictions, axis=0, weights=[0, 1, 2]) # let's do a weighted average for epochs (>1)\n# except:\n#     test_df.iloc[:, :] = np.average(history.test_predictions, axis=0, weights=[2**i for i in range(len(history.test_predictions))])\n#     #latest = history.test_predictions\n# \"\"\"\n# test_df0.iloc[:, :] = prediction[:len(test_df0)]\n\n# test_df0 = test_df0.stack().reset_index()\n\n# test_df0.insert(loc=0, column='ID', value=test_df0['Image'].astype(str) + \"_\" + test_df0['Diagnosis'])\n\n# test_df0 = test_df0.drop([\"Image\", \"Diagnosis\"], axis=1)\n\n# test_df0.to_csv('submission_test.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-06T17:30:21.205592Z","iopub.execute_input":"2021-07-06T17:30:21.205999Z","iopub.status.idle":"2021-07-06T17:30:21.215649Z","shell.execute_reply.started":"2021-07-06T17:30:21.205936Z","shell.execute_reply":"2021-07-06T17:30:21.214629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df0['Label'].columns","metadata":{"execution":{"iopub.status.busy":"2021-09-04T14:35:02.305907Z","iopub.execute_input":"2021-09-04T14:35:02.306278Z","iopub.status.idle":"2021-09-04T14:35:02.316924Z","shell.execute_reply.started":"2021-09-04T14:35:02.306211Z","shell.execute_reply":"2021-09-04T14:35:02.316228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score,roc_curve\nfrom sklearn import metrics\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2021-09-04T14:35:06.269412Z","iopub.execute_input":"2021-09-04T14:35:06.269735Z","iopub.status.idle":"2021-09-04T14:35:06.274159Z","shell.execute_reply.started":"2021-09-04T14:35:06.269679Z","shell.execute_reply":"2021-09-04T14:35:06.273248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# result val\ncol = ['any','epidural','intraparenchymal','intraventricular','subarachnoid','subdural']\ni = 1\npred = np.load('../input/intracranialhemorrhageserenext50output/pred_val.npz')['data']\ntrue = val_df0.values\n\nfor i in range(6):\n#     print(auc(true[:,i],pred[:75281,i]))\n    fpr, tpr, _ = metrics.roc_curve(true[:,i],pred[:75281,i])\n    auc = np.round(metrics.roc_auc_score(true[:,i],pred[:75281,i]),4)\n    plt.plot(fpr,tpr,label=f\"Class={col[i]}, auc=\"+str(auc))\n    plt.legend(prop={\"size\":12})\n    plt.xlim(0,0.4)\n    plt.ylim(0.6,1)\n    plt.ylabel('Sensitivity')\n    plt.xlabel('1 - Specificity')\n    plt.savefig(f'{col[i]}.jpg',dpi=250,bbox_inches = 'tight')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-04T14:35:17.774492Z","iopub.execute_input":"2021-09-04T14:35:17.774795Z","iopub.status.idle":"2021-09-04T14:35:20.439446Z","shell.execute_reply.started":"2021-09-04T14:35:17.774743Z","shell.execute_reply":"2021-09-04T14:35:20.438341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = weighted_log_loss(true[:],pred[:75281])\nb = a.eval(session=tf.compat.v1.Session())  \nnp.mean(b)","metadata":{"execution":{"iopub.status.busy":"2021-09-04T14:35:27.953323Z","iopub.execute_input":"2021-09-04T14:35:27.953637Z","iopub.status.idle":"2021-09-04T14:35:30.326276Z","shell.execute_reply.started":"2021-09-04T14:35:27.953584Z","shell.execute_reply":"2021-09-04T14:35:30.325466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# result train\ni = 1\npred = np.load('../input/intracranialhemorrhageserenext50output/pred_train.npz')['data']\ntrue = train_df0.values\nfor i in range(6):\n    print(roc_auc_score(true[:,i],pred[:677522,i]))","metadata":{"execution":{"iopub.status.busy":"2021-09-04T14:35:45.75936Z","iopub.execute_input":"2021-09-04T14:35:45.759681Z","iopub.status.idle":"2021-09-04T14:35:47.756904Z","shell.execute_reply.started":"2021-09-04T14:35:45.759626Z","shell.execute_reply":"2021-09-04T14:35:47.756001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = weighted_log_loss(true[:],pred[:677522])\nb = a.eval(session=tf.compat.v1.Session())  \nnp.mean(b)","metadata":{"execution":{"iopub.status.busy":"2021-09-04T14:35:47.758649Z","iopub.execute_input":"2021-09-04T14:35:47.759175Z","iopub.status.idle":"2021-09-04T14:35:48.678158Z","shell.execute_reply.started":"2021-09-04T14:35:47.759124Z","shell.execute_reply":"2021-09-04T14:35:48.677518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# result full\ni = 1\npred = np.concatenate((np.load('../input/intracranialhemorrhageserenext50output/pred_train.npz')['data'][:677522],\n            np.load('../input/intracranialhemorrhageserenext50output/pred_val.npz')['data'][:75281]),axis=0)\ntrue = np.concatenate((train_df0.values,val_df0.values),axis=0)\nfor i in range(6):\n    print(roc_auc_score(true[:,i],pred[:,i]))","metadata":{"execution":{"iopub.status.busy":"2021-09-04T14:35:56.794161Z","iopub.execute_input":"2021-09-04T14:35:56.794482Z","iopub.status.idle":"2021-09-04T14:35:58.343019Z","shell.execute_reply.started":"2021-09-04T14:35:56.794433Z","shell.execute_reply":"2021-09-04T14:35:58.342183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = weighted_log_loss(true[:],pred[:])\nb = a.eval(session=tf.compat.v1.Session())  \nnp.mean(b)","metadata":{"execution":{"iopub.status.busy":"2021-09-04T14:35:58.344708Z","iopub.execute_input":"2021-09-04T14:35:58.345176Z","iopub.status.idle":"2021-09-04T14:35:59.00585Z","shell.execute_reply.started":"2021-09-04T14:35:58.345123Z","shell.execute_reply":"2021-09-04T14:35:59.004948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.savez_compressed(f'train_idx.npz',data = train_idx)\nnp.savez_compressed(f'valid_idx.npz',data = valid_idx)","metadata":{"execution":{"iopub.status.busy":"2021-09-04T14:35:59.009565Z","iopub.execute_input":"2021-09-04T14:35:59.009823Z","iopub.status.idle":"2021-09-04T14:35:59.516068Z","shell.execute_reply.started":"2021-09-04T14:35:59.009776Z","shell.execute_reply":"2021-09-04T14:35:59.515342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# post process","metadata":{}},{"cell_type":"code","source":"i = 0\ndf_meta_train = pd.read_csv(f'../input/intracranialhemorrhagemetayousefzadeh/meta_train_{i}.csv')\ncol = df_meta_train.columns\ndf_meta_train = df_meta_train.values\n\nfor i in range(1,8):\n    df_meta_train = np.concatenate((df_meta_train,pd.read_csv(f'../input/intracranialhemorrhagemetayousefzadeh/meta_train_{i}.csv').values),axis=0)\n# df_meta_train = pd.DataFrame(df_meta_train[final_idx],columns=col)\\\ndf_meta_train = pd.DataFrame(df_meta_train,columns=col)\n\ndf_meta_train = df_meta_train[df_meta_train.duplicated()==False]\ndf_meta_train = df_meta_train.iloc[final_idx]","metadata":{"execution":{"iopub.status.busy":"2021-09-17T22:31:28.3053Z","iopub.execute_input":"2021-09-17T22:31:28.305646Z","iopub.status.idle":"2021-09-17T22:31:38.638601Z","shell.execute_reply.started":"2021-09-17T22:31:28.305587Z","shell.execute_reply":"2021-09-17T22:31:38.637849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_idx.shape,df_meta_train.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-17T22:25:34.765717Z","iopub.execute_input":"2021-09-17T22:25:34.766433Z","iopub.status.idle":"2021-09-17T22:25:34.780095Z","shell.execute_reply.started":"2021-09-17T22:25:34.766372Z","shell.execute_reply":"2021-09-17T22:25:34.777754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_meta_train[['type_0','type_1','type_2','type_3','type_4','type_5']].values.sum(axis=0)/len(df_meta_train)","metadata":{"execution":{"iopub.status.busy":"2021-09-17T22:25:38.846563Z","iopub.execute_input":"2021-09-17T22:25:38.846854Z","iopub.status.idle":"2021-09-17T22:25:39.252375Z","shell.execute_reply.started":"2021-09-17T22:25:38.8468Z","shell.execute_reply":"2021-09-17T22:25:39.251298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# p_df_train = np.unique(df_meta_train['patient_id'])\np_df_train","metadata":{"execution":{"iopub.status.busy":"2021-09-04T14:46:04.936992Z","iopub.execute_input":"2021-09-04T14:46:04.937314Z","iopub.status.idle":"2021-09-04T14:46:04.944418Z","shell.execute_reply.started":"2021-09-04T14:46:04.937261Z","shell.execute_reply":"2021-09-04T14:46:04.94346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p_df_train = np.unique(df_meta_train['patient_id'])\nnp.random.shuffle(p_df_train)\n\nnew_p = df_meta_train.iloc[df_meta_train['patient_id'].values==p_df_train[0]]\nnew_p = new_p.sort_values(by=['position_2']).values\n\nfor i in range(1,len(p_df_train)):\n    print(i)\n    new_p1 = df_meta_train.iloc[df_meta_train['patient_id'].values==p_df_train[i]]\n    new_p = np.concatenate((new_p,new_p1.sort_values(by=['position_2']).values),axis=0)","metadata":{"execution":{"iopub.status.idle":"2021-09-04T18:07:43.177235Z","shell.execute_reply.started":"2021-09-04T14:53:12.629279Z","shell.execute_reply":"2021-09-04T18:07:43.176122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir zip_100","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.ndimage import zoom\np_df_train = np.unique(df_meta_train['patient_id'])\n# np.random.shuffle(p_df_train)\nfor i in range(100):\n    new_p = df_meta_train.iloc[df_meta_train['patient_id'].values==p_df_train[i]]\n    new_p = new_p.sort_values(by=['position_2'])#.values\n    slices = new_p.id.values\n    dcm_3d = []\n    for slice_id in slices:\n        dcm = pydicom.dcmread(f'../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train/ID_{slice_id}.dcm')\n        dcm = dcm.pixel_array\n        dcm_3d.append(dcm)\n    dcm_3d = np.array(dcm_3d)\n    dcm_3d = dcm_3d.transpose(2,1,0)\n    dcm_3d = zoom(dcm_3d,(256/dcm_3d.shape[0],256/dcm_3d.shape[1],1),order=1)\n    np.savez(f'./zip_100/{i}.npz',data=dcm_3d)","metadata":{"execution":{"iopub.status.busy":"2021-09-17T22:42:33.991568Z","iopub.execute_input":"2021-09-17T22:42:33.991892Z","iopub.status.idle":"2021-09-17T22:44:03.162028Z","shell.execute_reply.started":"2021-09-17T22:42:33.991835Z","shell.execute_reply":"2021-09-17T22:44:03.161202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nn_df = pd.DataFrame(new_p,columns = df_meta_train.columns)\nnn_df","metadata":{"execution":{"iopub.status.busy":"2021-09-17T22:26:59.312284Z","iopub.execute_input":"2021-09-17T22:26:59.312582Z","iopub.status.idle":"2021-09-17T22:26:59.340996Z","shell.execute_reply.started":"2021-09-17T22:26:59.312531Z","shell.execute_reply":"2021-09-17T22:26:59.339713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nn_df = pd.DataFrame(new_p,columns = df_meta_train.columns)\nnn_df.to_csv('meta_patient_homorrhage.csv',index=False)\nnn_df","metadata":{"execution":{"iopub.status.busy":"2021-09-04T18:20:00.18867Z","iopub.execute_input":"2021-09-04T18:20:00.188984Z","iopub.status.idle":"2021-09-04T18:20:13.645971Z","shell.execute_reply.started":"2021-09-04T18:20:00.188932Z","shell.execute_reply":"2021-09-04T18:20:13.643935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p_df_train = np.unique(df_meta_train['patient_id'])\nnew_p = df_meta_train.iloc[df_meta_train['patient_id'].values==p_df_train[2]]\nnew_p = new_p.sort_values(by=['position_2'])\nfor i in new_p['id']:\n    plt.imshow(_read(train_images_dir+f'ID_{i}.dcm', (256, 256)), cmap=plt.cm.bone)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-04T14:38:54.64845Z","iopub.execute_input":"2021-09-04T14:38:54.648762Z","iopub.status.idle":"2021-09-04T14:39:02.763779Z","shell.execute_reply.started":"2021-09-04T14:38:54.648708Z","shell.execute_reply":"2021-09-04T14:39:02.763106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# full data\ntrain_meta_df0 = df_meta_train.iloc[train_idx]\nval_meta_df0 = df_meta_train.iloc[valid_idx]\ntrain_meta_df0.shape,val_meta_df0.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-06T17:35:29.456768Z","iopub.execute_input":"2021-07-06T17:35:29.457063Z","iopub.status.idle":"2021-07-06T17:35:32.762054Z","shell.execute_reply.started":"2021-07-06T17:35:29.45701Z","shell.execute_reply":"2021-07-06T17:35:32.76095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_meta_df0.columns","metadata":{"execution":{"iopub.status.busy":"2021-07-06T17:35:32.765206Z","iopub.execute_input":"2021-07-06T17:35:32.765596Z","iopub.status.idle":"2021-07-06T17:35:32.773026Z","shell.execute_reply.started":"2021-07-06T17:35:32.765531Z","shell.execute_reply":"2021-07-06T17:35:32.771912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_meta = np.concatenate((train_meta_df0.values, val_meta_df0.values),axis=0)\ndf_df_full_meta = pd.DataFrame(full_meta,columns = train_meta_df0.columns)\nfull_pred = np.concatenate((np.load('../input/intracranialhemorrhageserenext50output/pred_train.npz')['data'][:677522],\n            np.load('../input/intracranialhemorrhageserenext50output/pred_val.npz')['data'][:75281]),axis=0)\n\ndf_df_full_meta = df_df_full_meta[['id','type_5', 'type_0', 'type_1', 'type_2', 'type_3', 'type_4',\n       'patient_id', 'position_0', 'position_1', 'position_2', 'orientation_0',\n       'orientation_1', 'orientation_2', 'orientation_3', 'orientation_4',\n       'orientation_5']]\n\nfull_meta = df_df_full_meta.values\n\nfor i,j in enumerate(['p1','p2','p3','p4','p5','p6']):\n    df_df_full_meta[j] = full_pred[:,i]","metadata":{"execution":{"iopub.status.busy":"2021-07-06T17:35:32.77479Z","iopub.execute_input":"2021-07-06T17:35:32.775101Z","iopub.status.idle":"2021-07-06T17:35:38.790142Z","shell.execute_reply.started":"2021-07-06T17:35:32.775045Z","shell.execute_reply":"2021-07-06T17:35:38.788853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# result full\ntrue = np.array(full_meta[:,[1,2,3,4,5,6]],int)\nfor i in range(6):\n    print(roc_auc_score(true[:,i],full_pred[:,i]))","metadata":{"execution":{"iopub.status.busy":"2021-07-06T17:35:38.791995Z","iopub.execute_input":"2021-07-06T17:35:38.792395Z","iopub.status.idle":"2021-07-06T17:35:42.493022Z","shell.execute_reply.started":"2021-07-06T17:35:38.792323Z","shell.execute_reply":"2021-07-06T17:35:42.491623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# jj = -125000\n# print(df.iloc[jj])\n# print('###############################')\n# df_meta_train.iloc[jj]","metadata":{"execution":{"iopub.status.busy":"2021-07-06T17:35:42.495007Z","iopub.execute_input":"2021-07-06T17:35:42.495445Z","iopub.status.idle":"2021-07-06T17:35:42.499926Z","shell.execute_reply.started":"2021-07-06T17:35:42.495367Z","shell.execute_reply":"2021-07-06T17:35:42.498771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# labels, counts = np.unique(zz,return_counts=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-06T17:35:42.501762Z","iopub.execute_input":"2021-07-06T17:35:42.502211Z","iopub.status.idle":"2021-07-06T17:35:42.51311Z","shell.execute_reply.started":"2021-07-06T17:35:42.502136Z","shell.execute_reply":"2021-07-06T17:35:42.51177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# vvv0 = df.iloc[train_idx]\n# vvv1 = df_meta_train.iloc[train_idx]\n\n# jj = 2\n# print(vvv0.iloc[jj])\n# print('###############################')\n# vvv1.iloc[jj]","metadata":{"execution":{"iopub.status.busy":"2021-07-06T17:35:42.514638Z","iopub.execute_input":"2021-07-06T17:35:42.515002Z","iopub.status.idle":"2021-07-06T17:35:42.524619Z","shell.execute_reply.started":"2021-07-06T17:35:42.514951Z","shell.execute_reply":"2021-07-06T17:35:42.523686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.ndimage import gaussian_filter as gf","metadata":{"execution":{"iopub.status.busy":"2021-07-06T17:35:42.526298Z","iopub.execute_input":"2021-07-06T17:35:42.526677Z","iopub.status.idle":"2021-07-06T17:35:42.536532Z","shell.execute_reply.started":"2021-07-06T17:35:42.52662Z","shell.execute_reply":"2021-07-06T17:35:42.535066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p_df_train = np.unique(df_df_full_meta['patient_id'])\nnew_p = df_df_full_meta.iloc[df_df_full_meta['patient_id'].values==p_df_train[5]]\nnew_p = new_p.sort_values(by=['position_2'])\n\nplt.plot(new_p['position_2'],new_p.iloc[:,1+16])\nplt.plot(new_p['position_2'],new_p.iloc[:,1],color='g')\nplt.plot(new_p['position_2'],gf(new_p.iloc[:,1+16],1),'r')\ntitle = np.array(np.max(new_p.iloc[:,1:7].values,axis=0),str)\nplt.title(title[0]+title[1]+title[2]+title[3]+title[4]+title[5])","metadata":{"execution":{"iopub.status.busy":"2021-07-06T17:35:42.538308Z","iopub.execute_input":"2021-07-06T17:35:42.538768Z","iopub.status.idle":"2021-07-06T17:35:45.049045Z","shell.execute_reply.started":"2021-07-06T17:35:42.538691Z","shell.execute_reply":"2021-07-06T17:35:45.048028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p_df_train = np.unique(df_df_full_meta['patient_id'])\nnew_p = df_df_full_meta.iloc[df_df_full_meta['patient_id'].values==p_df_train[5]]\nnew_p = new_p.sort_values(by=['position_2'])\n\n# plt.plot(new_p['position_2'],)\nplt.plot(new_p['position_2'],gf(new_p.iloc[:,1+16],2)-new_p.iloc[:,1+16],'r')\ntitle = np.array(np.max(new_p.iloc[:,1:7].values,axis=0),str)\nplt.title(title[0]+title[1]+title[2]+title[3]+title[4]+title[5])","metadata":{"execution":{"iopub.status.busy":"2021-07-06T17:35:45.050323Z","iopub.execute_input":"2021-07-06T17:35:45.050729Z","iopub.status.idle":"2021-07-06T17:35:47.568053Z","shell.execute_reply.started":"2021-07-06T17:35:45.050688Z","shell.execute_reply":"2021-07-06T17:35:47.567082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p_df_train = np.unique(df_df_full_meta['patient_id'])\nall_result = []\n\n# [0,0.4,0.5,0.6,0.7,0.8,1,1.2,1.4,1.6,2,2.5]\n\nfor sigma in [1.6,2,2.5]:\n    gf_out = []\n    new_df = []\n\n    g1,g2,g3,g4,g5,g6 = [[],[],[],[],[],[]]\n    for i,j in enumerate(p_df_train[:]):\n    #     print(i)\n        new_p = df_df_full_meta.iloc[df_df_full_meta['patient_id'].values==j]\n        new_p = new_p.sort_values(by=['position_2'])\n        g1+=list(gf(new_p.iloc[:,17],sigma))\n        g2+=list(gf(new_p.iloc[:,18],sigma))\n        g3+=list(gf(new_p.iloc[:,19],sigma))\n        g4+=list(gf(new_p.iloc[:,20],sigma))\n        g5+=list(gf(new_p.iloc[:,21],sigma))\n        g6+=list(gf(new_p.iloc[:,22],sigma))\n\n        new_df+=list(new_p.values)\n\n    g1 = np.array(g1)\n    g2 = np.array(g2)\n    g3 = np.array(g3)\n    g4 = np.array(g4)\n    g5 = np.array(g5)\n    g6 = np.array(g6)\n    new_df = np.array(new_df)\n    new_df = pd.DataFrame(new_df,columns = new_p.columns)\n    result_gf = [metrics.roc_auc_score(np.array(new_df.iloc[:,1].values,int),g1),\n                 metrics.roc_auc_score(np.array(new_df.iloc[:,2].values,int),g2),\n                 metrics.roc_auc_score(np.array(new_df.iloc[:,3].values,int),g3),\n                 metrics.roc_auc_score(np.array(new_df.iloc[:,4].values,int),g4),\n                 metrics.roc_auc_score(np.array(new_df.iloc[:,5].values,int),g5),\n                 metrics.roc_auc_score(np.array(new_df.iloc[:,6].values,int),g6)]\n    \n    print(sigma)\n    print(result_gf)\n    print('#########################')\n    all_result.append(result_gf)\n    \nnp.save('all_result.npy',all_result)","metadata":{"execution":{"iopub.status.busy":"2021-07-06T17:35:47.56972Z","iopub.execute_input":"2021-07-06T17:35:47.570044Z","iopub.status.idle":"2021-07-06T17:36:01.111876Z","shell.execute_reply.started":"2021-07-06T17:35:47.56999Z","shell.execute_reply":"2021-07-06T17:36:01.110468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}