{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install pydicom kornia opencv-python scikit-image pyarrow","metadata":{"execution":{"iopub.status.busy":"2022-03-13T22:40:13.796746Z","iopub.execute_input":"2022-03-13T22:40:13.796998Z","iopub.status.idle":"2022-03-13T22:40:20.925219Z","shell.execute_reply.started":"2022-03-13T22:40:13.79697Z","shell.execute_reply":"2022-03-13T22:40:20.924013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from fastai.basics import *\nfrom fastai.callback.all import *\nfrom fastai.vision.all import *\nfrom fastai.medical.imaging import *\n\nimport pydicom\n\nseed = 42  # random number generator seed => macht Zufall wiederholbar","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-14T02:06:25.356512Z","iopub.execute_input":"2022-03-14T02:06:25.356822Z","iopub.status.idle":"2022-03-14T02:06:29.641299Z","shell.execute_reply.started":"2022-03-14T02:06:25.356751Z","shell.execute_reply":"2022-03-14T02:06:29.640163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = {}   # hier werden manuell gelabelte Beispiele gespeichert\nverlauf = []  # zur Überprüfung etwaiger Fehler","metadata":{"execution":{"iopub.status.busy":"2022-03-14T02:06:29.643461Z","iopub.execute_input":"2022-03-14T02:06:29.643844Z","iopub.status.idle":"2022-03-14T02:06:29.649988Z","shell.execute_reply.started":"2022-03-14T02:06:29.643806Z","shell.execute_reply":"2022-03-14T02:06:29.648646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Daten anschauen","metadata":{}},{"cell_type":"code","source":"path_data = Path(\"../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train/\")\nassert path_data.exists()","metadata":{"execution":{"iopub.status.busy":"2022-03-14T02:06:29.651183Z","iopub.execute_input":"2022-03-14T02:06:29.65142Z","iopub.status.idle":"2022-03-14T02:06:29.669939Z","shell.execute_reply.started":"2022-03-14T02:06:29.651392Z","shell.execute_reply":"2022-03-14T02:06:29.668765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fastai überprüft normalerweise ob die Dateien wirklich korrekt sind\n# das macht das Laden der DICOMs ziemlich langsam\n# zudem verwenden wir hier ein professionell erstelltes Datenset\n# daher beschleunigen wir das Laden der Daten indem wir hier die eingebauten Funktionen von fastai durch unsere eigenen ersetzen\n# zudem bauen wir ein Limit ein falls wir nicht alle DICOMs laden wollen\n\ndef get_files(path, extensions=None, folders=None, followlinks=True, limit=None):\n    \"Get all the files in `path` with optional `extensions`, optionally with `recurse`, only in `folders`, if specified.\"\n    path = Path(path)\n    folders=L(folders)\n    extensions = setify(extensions)\n    extensions = {e.lower() for e in extensions}    \n    \n#   f = [o.name for o in os.scandir(path) if o.is_file()]  # hier überprüft das Original ob es wirklich Dateien sind\n    f = [o.name for o in os.scandir(path)]\n    if limit:\n        f = f[:limit]\n            \n    res = _get_files(path, f, extensions)\n    \n    return L(res)\n\n\ndef _get_files(p, fs, extensions=None):\n    p = Path(p)\n    res = [p/f for f in fs if not f.startswith('.')\n           and ((not extensions) or f'.{f.split(\".\")[-1].lower()}' in extensions)]\n    return res\n\n\ndef get_dicom_files(path, folders=None, limit=None):\n    \"Get dicom files in `path` recursively, only in `folders`, if specified.\"\n    return get_files(path, extensions=[\".dcm\",\".dicom\"], folders=folders, limit=limit)","metadata":{"execution":{"iopub.status.busy":"2022-03-14T02:06:29.672005Z","iopub.execute_input":"2022-03-14T02:06:29.672633Z","iopub.status.idle":"2022-03-14T02:06:29.685785Z","shell.execute_reply.started":"2022-03-14T02:06:29.672599Z","shell.execute_reply":"2022-03-14T02:06:29.684559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n# das laden aller DICOMs dauert trotzdem etwas\ndicoms = get_dicom_files(path_data, limit=None)","metadata":{"execution":{"iopub.status.busy":"2022-03-14T02:06:29.686883Z","iopub.execute_input":"2022-03-14T02:06:29.687535Z","iopub.status.idle":"2022-03-14T02:06:53.843329Z","shell.execute_reply.started":"2022-03-14T02:06:29.687502Z","shell.execute_reply":"2022-03-14T02:06:53.84209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dicoms","metadata":{"execution":{"iopub.status.busy":"2022-03-14T02:06:53.845151Z","iopub.execute_input":"2022-03-14T02:06:53.845585Z","iopub.status.idle":"2022-03-14T02:06:53.855757Z","shell.execute_reply.started":"2022-03-14T02:06:53.845544Z","shell.execute_reply":"2022-03-14T02:06:53.854352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# hier speichern wir DICOMS die wir schon mal gesehen haben damit wir nicht eines doppelt erwischen\nseen_dicoms = set()","metadata":{"execution":{"iopub.status.busy":"2022-03-14T02:06:53.858554Z","iopub.execute_input":"2022-03-14T02:06:53.859325Z","iopub.status.idle":"2022-03-14T02:06:53.869254Z","shell.execute_reply.started":"2022-03-14T02:06:53.859266Z","shell.execute_reply":"2022-03-14T02:06:53.868686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# diese Funktion wählt jedes Mal eine neue zufällige DICOM Datei aus\ndef get_random_dicom(dicoms):\n    dcm = random.choice(dicoms)\n    \n    while dcm in seen_dicoms:\n        # neues random dicom holen\n        dcm = random.sample(dicoms)\n    \n    seen_dicoms.add(dcm)   \n    return dcm","metadata":{"execution":{"iopub.status.busy":"2022-03-14T02:06:53.870487Z","iopub.execute_input":"2022-03-14T02:06:53.871428Z","iopub.status.idle":"2022-03-14T02:06:53.883542Z","shell.execute_reply.started":"2022-03-14T02:06:53.871391Z","shell.execute_reply":"2022-03-14T02:06:53.882805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Datenset ausbalanzieren\nim ursprünglichen Datenset sind nur ca 5% der Bilder von Gehirnblutungen  \nhier gleichen wir das aus indem wir genug nicht-Blutungen raus filtern bis wir ein 50:50 Verhältnis haben  \n(Downsampling)","metadata":{}},{"cell_type":"code","source":"path_csv = Path(\"../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train.csv\")\n\ndf = pd.read_csv(path_csv)","metadata":{"execution":{"iopub.status.busy":"2022-03-14T02:06:53.884679Z","iopub.execute_input":"2022-03-14T02:06:53.885301Z","iopub.status.idle":"2022-03-14T02:06:58.470282Z","shell.execute_reply.started":"2022-03-14T02:06:53.88527Z","shell.execute_reply":"2022-03-14T02:06:58.469515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[[\"filename\", \"exact_label\"]] = df[\"ID\"].str.rsplit(\"_\", 1, expand=True)\ndf.set_index(\"filename\", inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-03-14T02:06:58.471418Z","iopub.execute_input":"2022-03-14T02:06:58.471667Z","iopub.status.idle":"2022-03-14T02:07:14.202391Z","shell.execute_reply.started":"2022-03-14T02:06:58.471638Z","shell.execute_reply":"2022-03-14T02:07:14.200518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_0 = (df[df[\"Label\"] == 0])\ndf_1 = (df[df[\"Label\"] == 1])\n\nn = len(df_1)","metadata":{"execution":{"iopub.status.busy":"2022-03-14T02:07:14.203682Z","iopub.execute_input":"2022-03-14T02:07:14.203898Z","iopub.status.idle":"2022-03-14T02:07:15.047339Z","shell.execute_reply.started":"2022-03-14T02:07:14.203868Z","shell.execute_reply":"2022-03-14T02:07:15.045939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"samples = pd.concat([df_1, df_0.sample(n=n, random_state=seed)])","metadata":{"execution":{"iopub.status.busy":"2022-03-14T02:07:15.048674Z","iopub.execute_input":"2022-03-14T02:07:15.049453Z","iopub.status.idle":"2022-03-14T02:07:15.506916Z","shell.execute_reply.started":"2022-03-14T02:07:15.049418Z","shell.execute_reply":"2022-03-14T02:07:15.505562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list(samples[\"Label\"].value_counts().items())","metadata":{"execution":{"iopub.status.busy":"2022-03-14T02:07:15.508222Z","iopub.execute_input":"2022-03-14T02:07:15.508526Z","iopub.status.idle":"2022-03-14T02:07:15.522696Z","shell.execute_reply.started":"2022-03-14T02:07:15.508491Z","shell.execute_reply":"2022-03-14T02:07:15.521782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Blutung ja/nein:    Anzahl DICOMs\")\nfor k, v in samples[\"Label\"].value_counts().items():\n    print(f\" {k}                   {v}\")","metadata":{"execution":{"iopub.status.busy":"2022-03-14T02:07:15.525457Z","iopub.execute_input":"2022-03-14T02:07:15.525746Z","iopub.status.idle":"2022-03-14T02:07:15.541642Z","shell.execute_reply.started":"2022-03-14T02:07:15.525716Z","shell.execute_reply":"2022-03-14T02:07:15.540751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dicom zufällig auswählen und labeln","metadata":{"execution":{"iopub.status.busy":"2022-03-13T18:53:40.961527Z","iopub.execute_input":"2022-03-13T18:53:40.961806Z","iopub.status.idle":"2022-03-13T18:53:40.965748Z","shell.execute_reply.started":"2022-03-13T18:53:40.961777Z","shell.execute_reply":"2022-03-13T18:53:40.965048Z"}}},{"cell_type":"code","source":"scales = [\n    dicom_windows.brain,  # W:80, L:40\n#     True,  # normalized\n#     dicom_windows.subdural,  # W:254, L:100\n]\n\ntitles = [\n    \"Brain Window\",\n#     \"Normalized\",\n#     \"Subdural Window\"\n]","metadata":{"execution":{"iopub.status.busy":"2022-03-14T02:07:15.542837Z","iopub.execute_input":"2022-03-14T02:07:15.543171Z","iopub.status.idle":"2022-03-14T02:07:15.551984Z","shell.execute_reply.started":"2022-03-14T02:07:15.543135Z","shell.execute_reply":"2022-03-14T02:07:15.550614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Daten labeln","metadata":{}},{"cell_type":"code","source":"# hier holen wir uns ein neues zufälliges DICOM\nsample = get_random_dicom(dicoms)\nsample.name","metadata":{"execution":{"iopub.status.busy":"2022-03-14T02:07:15.553626Z","iopub.execute_input":"2022-03-14T02:07:15.553872Z","iopub.status.idle":"2022-03-14T02:07:15.570953Z","shell.execute_reply.started":"2022-03-14T02:07:15.553837Z","shell.execute_reply":"2022-03-14T02:07:15.570415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DICOM anzeigen\nbild_größe = 12\n\nfor scale, axis, title in zip(scales, subplots(len(scales),1,imsize=bild_größe)[1].flat, titles):\n    dcm = sample.dcmread()\n    dcm.show(scale=scale, ax=axis, title=title)","metadata":{"execution":{"iopub.status.busy":"2022-03-14T02:07:15.571774Z","iopub.execute_input":"2022-03-14T02:07:15.572194Z","iopub.status.idle":"2022-03-14T02:07:16.015621Z","shell.execute_reply.started":"2022-03-14T02:07:15.572166Z","shell.execute_reply":"2022-03-14T02:07:16.01459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ist eine Blutung\nlabels[sample.name] = 1\nverlauf.append(f\"{sample.name}=1 \")","metadata":{"execution":{"iopub.status.busy":"2022-03-14T02:07:16.016977Z","iopub.execute_input":"2022-03-14T02:07:16.017205Z","iopub.status.idle":"2022-03-14T02:07:16.022294Z","shell.execute_reply.started":"2022-03-14T02:07:16.017167Z","shell.execute_reply":"2022-03-14T02:07:16.021404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ist KEINE Blutung\nlabels[sample.name] = 0\nverlauf.append(f\"{sample.name}=0 \")","metadata":{"execution":{"iopub.status.busy":"2022-03-14T02:07:16.023254Z","iopub.execute_input":"2022-03-14T02:07:16.023462Z","iopub.status.idle":"2022-03-14T02:07:16.041053Z","shell.execute_reply.started":"2022-03-14T02:07:16.023432Z","shell.execute_reply":"2022-03-14T02:07:16.040306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Bisher gelabelt: {len(labels)} DICOMs\")","metadata":{"execution":{"iopub.status.busy":"2022-03-14T02:07:16.71957Z","iopub.execute_input":"2022-03-14T02:07:16.720274Z","iopub.status.idle":"2022-03-14T02:07:16.725782Z","shell.execute_reply.started":"2022-03-14T02:07:16.720177Z","shell.execute_reply":"2022-03-14T02:07:16.724167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels","metadata":{"execution":{"iopub.status.busy":"2022-03-14T02:07:18.577365Z","iopub.execute_input":"2022-03-14T02:07:18.577867Z","iopub.status.idle":"2022-03-14T02:07:18.585564Z","shell.execute_reply.started":"2022-03-14T02:07:18.577829Z","shell.execute_reply":"2022-03-14T02:07:18.584545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"verlauf[-5:]","metadata":{"execution":{"iopub.status.busy":"2022-03-14T02:07:21.586663Z","iopub.execute_input":"2022-03-14T02:07:21.586949Z","iopub.status.idle":"2022-03-14T02:07:21.592914Z","shell.execute_reply.started":"2022-03-14T02:07:21.586921Z","shell.execute_reply":"2022-03-14T02:07:21.592139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Labels speichern","metadata":{}},{"cell_type":"code","source":"# Labels speichern\n\nname_student = \"\"\nassert name_student, \"Bitte einen gültigen Namen ausfüllen\"\n\npd.Series(labels).to_csv(f\"./{name_student}.csv\", header=False)\n\nprint(\"Nach dem Speichern nicht vergessen die Daten auch runter zu laden!\")","metadata":{"execution":{"iopub.status.busy":"2022-03-14T02:11:17.708858Z","iopub.execute_input":"2022-03-14T02:11:17.709112Z","iopub.status.idle":"2022-03-14T02:11:17.837695Z","shell.execute_reply.started":"2022-03-14T02:11:17.709083Z","shell.execute_reply":"2022-03-14T02:11:17.836606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}