{"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":45867,"databundleVersionId":6924515,"sourceType":"competition"},{"sourceId":6640479,"sourceType":"datasetVersion","datasetId":3833517},{"sourceId":6746686,"sourceType":"datasetVersion","datasetId":3884593},{"sourceId":6827935,"sourceType":"datasetVersion","datasetId":3926155},{"sourceId":6874344,"sourceType":"datasetVersion","datasetId":3950227},{"sourceId":7259757,"sourceType":"datasetVersion","datasetId":4207197},{"sourceId":7270449,"sourceType":"datasetVersion","datasetId":4214573},{"sourceId":147635265,"sourceType":"kernelVersion"},{"sourceId":3741,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":2668}],"dockerImageVersionId":30627,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.12"},"papermill":{"default_parameters":{},"duration":25.535433,"end_time":"2023-12-24T09:26:12.572319","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2023-12-24T09:25:47.036886","version":"2.4.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"use datasets, new code","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport math\nimport copy\nimport time\nimport random\nimport glob\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import lr_scheduler\nimport torchvision\nimport timm\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold\nimport joblib\nfrom tqdm import tqdm\nfrom collections import defaultdict\nfrom PIL import Image\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nos.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"\n\n# Configuration\nCONFIG = {\n    \"seed\": 40,\n    \"img_size\": 475,\n    \"model_name\": \"tf_efficientnet_l2_ns_475\",\n    \"num_classes\": 5,\n    \"valid_batch_size\": 2,\n    \"device\": torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\"),\n}\n\n# Set seed for reproducibility\ndef set_seed(seed=42):\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nset_seed(CONFIG['seed'])\n\n# Directory paths and file locations\nROOT_DIR = '/kaggle/input/UBC-OCEAN'\nTEST_DIR = '/kaggle/input/UBC-OCEAN/test_thumbnails'\nALT_TEST_DIR = '/kaggle/input/UBC-OCEAN/test_images'\nLABEL_ENCODER_BIN = \"/kaggle/input/ubcpytorchwith-classweights-training-fold1of5/label_encoder.pkl\"\n\n# Function to get test file path\ndef get_test_file_path(image_id):\n    if os.path.exists(f\"{TEST_DIR}/{image_id}_thumbnail.png\"):\n        return f\"{TEST_DIR}/{image_id}_thumbnail.png\"\n    else:\n        return f\"{ALT_TEST_DIR}/{image_id}.png\"\n\n# Reading test data\ndf = pd.read_csv(f\"{ROOT_DIR}/test.csv\")\ndf['file_path'] = df['image_id'].apply(get_test_file_path)\ndf['label'] = 0\n\n# Reading sample submission data\ndf_sub = pd.read_csv(f\"{ROOT_DIR}/sample_submission.csv\")\n\n# Loading label encoder\nencoder = joblib.load(LABEL_ENCODER_BIN)\n\n# Data transformations\ndata_transforms = {\n    \"valid\": A.Compose([\n        A.Resize(CONFIG['img_size'], CONFIG['img_size']),\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n        ToTensorV2()], p=1.)\n}\n\n# UBCDataset class\nclass UBCDataset(Dataset):\n    def __init__(self, df, transforms=None):\n        self.df = df\n        self.file_names = df['file_path'].values\n        self.labels = df['label'].values\n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        img_path = self.file_names[index]\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        if self.transforms:\n            img = self.transforms(image=img)[\"image\"]\n            \n        return {'image': img, 'label': torch.tensor(self.labels[index], dtype=torch.long)}\n\n# Custom model class\nclass UBCModel(nn.Module):\n    def __init__(self, model_name, num_classes, local_pretrained_path=None):\n        super(UBCModel, self).__init__()\n        self.model = timm.create_model(model_name, pretrained=False)\n        in_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(in_features, num_classes)\n        self.dropout = nn.Dropout(0.5)\n\n        if local_pretrained_path is not None:\n            self.load_local_weights(local_pretrained_path)\n\n    def load_local_weights(self, weights_path):\n        try:\n            state_dict = torch.load(weights_path, map_location=CONFIG['device'])\n            state_dict = {k.replace('module.', ''): v for k, v in state_dict.items()}\n            self.model.load_state_dict(state_dict)\n        except Exception as e:\n            print(f\"Error loading local weights: {e}\")\n\n    def forward(self, images):\n        x = self.model(images)\n        x = self.dropout(x)\n        return x\n\n# Model initialization\nlocal_weights_path = '/kaggle/input/tf-efficientnet/pytorch/tf-efficientnet-l2-ns-475/1/tf_efficientnet_l2_ns_475-bebbd00a.pth'\nmodel = UBCModel(CONFIG['model_name'], CONFIG['num_classes'], local_pretrained_path=local_weights_path)\nmodel.to(CONFIG['device'])\n\n# DataLoader\ntest_dataset = UBCDataset(df, transforms=data_transforms[\"valid\"])\ntest_loader = DataLoader(test_dataset, batch_size=CONFIG['valid_batch_size'], num_workers=2, shuffle=False, pin_memory=True)\n\n# Inference\npreds = []\nwith torch.no_grad():\n    for step, data in tqdm(enumerate(test_loader), total=len(test_loader)):\n        images = data['image'].to(CONFIG[\"device\"], dtype=torch.float)\n        outputs = model(images)\n        _, predicted = torch.max(outputs, 1)\n        preds.append(predicted.detach().cpu().numpy())\n\npreds = np.concatenate(preds).flatten()\npred_labels = encoder.inverse_transform(preds)\n\n# Update submission DataFrame\ndf_sub[\"label\"] = pred_labels\ndf_sub.to_csv(\"submission.csv\", index=False)\n\n# Print the submission DataFrame\nprint(df_sub)\n","metadata":{"execution":{"iopub.status.busy":"2024-01-03T00:07:48.776104Z","iopub.execute_input":"2024-01-03T00:07:48.776941Z","iopub.status.idle":"2024-01-03T00:08:16.273283Z","shell.execute_reply.started":"2024-01-03T00:07:48.776902Z","shell.execute_reply":"2024-01-03T00:08:16.27216Z"},"trusted":true},"execution_count":null,"outputs":[]}]}