{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":56537,"databundleVersionId":8015876,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import random, sys, gc, warnings, math\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nimport matplotlib.pyplot as plt\nfrom time import time\nimport warnings\n\nwarnings.filterwarnings('ignore', category=FutureWarning)\nt0 = time()\nnp.random.seed(42)\nrandom.seed(42)\nmin_std = 1e-8\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nDEBUGGING = False","metadata":{"execution":{"iopub.status.busy":"2024-04-26T05:59:18.918235Z","iopub.execute_input":"2024-04-26T05:59:18.918622Z","iopub.status.idle":"2024-04-26T05:59:18.985297Z","shell.execute_reply.started":"2024-04-26T05:59:18.918592Z","shell.execute_reply":"2024-04-26T05:59:18.984347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pytorch-base Feed-Forward Neural Network (FFNN)\nThis is a starting notebook to train a FFNN for regression\n\n### To-do's\n- Try different activations (Mish, Swish), normalization (layer, instance), regularization (dropout, L2), weight initialization (He...)\n- Experiment different model sizes: hidden-size, n_layers...\n- Try different architectures: residual connections, attention?\n\n### Comment\nFeel free to comment with any fixes, ideas, or optimizations you think could enhance this notebook\n","metadata":{}},{"cell_type":"code","source":"# load train data\nif DEBUGGING:\n    n_rows = 10000\nelse:\n    n_rows = 1000000\ndf = pd.read_csv(\"/kaggle/input/leap-atmospheric-physics-ai-climsim/train.csv\", nrows=n_rows)\nx = df.iloc[:,1:557].to_numpy().astype(np.float32)\ny = df.iloc[:,557:].to_numpy().astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2024-04-26T10:49:00.750952Z","iopub.execute_input":"2024-04-26T10:49:00.751743Z","iopub.status.idle":"2024-04-26T10:49:01.07754Z","shell.execute_reply.started":"2024-04-26T10:49:00.751709Z","shell.execute_reply":"2024-04-26T10:49:01.076387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read test\nif not DEBUGGING:\n    df = pd.read_csv(\"/kaggle/input/leap-atmospheric-physics-ai-climsim/test.csv\")\n    xt = df.iloc[:,1:557].to_numpy().astype(np.float32)\n    del df\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-26T05:55:44.925839Z","iopub.execute_input":"2024-04-26T05:55:44.926186Z","iopub.status.idle":"2024-04-26T05:55:49.443874Z","shell.execute_reply.started":"2024-04-26T05:55:44.926161Z","shell.execute_reply":"2024-04-26T05:55:49.442608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# norm X\nmx = x.mean(axis=0)\nsx = np.maximum(x.std(axis=0), min_std)\nx = (x - mx.reshape(1,-1)) / sx.reshape(1,-1)\nif not DEBUGGING:\n    xt = (xt - mx.reshape(1,-1)) / sx.reshape(1,-1)\n\n# norm Y\nmy = y.mean(axis=0)\nsy = np.maximum(np.sqrt((y*y).mean(axis=0)), min_std)\ny = (y - my.reshape(1,-1)) / sy.reshape(1,-1)","metadata":{"execution":{"iopub.status.busy":"2024-04-26T05:57:21.559625Z","iopub.execute_input":"2024-04-26T05:57:21.560248Z","iopub.status.idle":"2024-04-26T05:57:21.598635Z","shell.execute_reply.started":"2024-04-26T05:57:21.560216Z","shell.execute_reply":"2024-04-26T05:57:21.597461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FFNN(nn.Module):\n    def __init__(self, input_size, hidden_sizes, output_size):\n        super(FFNN, self).__init__()\n        \n        # Initialize the layers\n        layers = []\n        previous_size = input_size\n        for hidden_size in hidden_sizes:\n            layers.append(nn.Linear(previous_size, hidden_size))\n            layers.append(nn.LayerNorm(hidden_size))  # Normalization layer\n            layers.append(nn.LeakyReLU(inplace=True))        # Activation\n            layers.append(nn.Dropout(p=0.1))            # Dropout for regularization\n            previous_size = hidden_size\n        \n        # Output layer - no dropout, no activation function\n        layers.append(nn.Linear(previous_size, output_size))\n        \n        # Register all layers\n        self.layers = nn.Sequential(*layers)\n\n    def forward(self, x):\n        return self.layers(x)","metadata":{"execution":{"iopub.status.busy":"2024-04-26T05:56:59.937237Z","iopub.execute_input":"2024-04-26T05:56:59.938143Z","iopub.status.idle":"2024-04-26T05:56:59.945331Z","shell.execute_reply.started":"2024-04-26T05:56:59.938106Z","shell.execute_reply":"2024-04-26T05:56:59.944387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class NumpyDataset(Dataset):\n    def __init__(self, x, y):\n        \"\"\"\n        Initialize with NumPy arrays.\n        \"\"\"\n        assert x.shape[0] == y.shape[0], \"Features and labels must have the same number of samples\"\n        self.x = x\n        self.y = y\n\n    def __len__(self):\n        \"\"\"\n        Total number of samples.\n        \"\"\"\n        return self.x.shape[0]\n\n    def __getitem__(self, index):\n        \"\"\"\n        Generate one sample of data.\n        \"\"\"\n        # Convert the data to tensors when requested\n        return torch.from_numpy(self.x[index]).float().to(device), torch.from_numpy(self.y[index]).float().to(device)\n\n","metadata":{"execution":{"iopub.status.busy":"2024-04-26T05:59:26.612388Z","iopub.execute_input":"2024-04-26T05:59:26.613248Z","iopub.status.idle":"2024-04-26T05:59:26.619619Z","shell.execute_reply.started":"2024-04-26T05:59:26.613211Z","shell.execute_reply":"2024-04-26T05:59:26.618723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = NumpyDataset(x, y)\n\ntrain_size = int(0.9 * len(dataset))\nval_size = len(dataset) - train_size\ntrain_dataset, val_dataset = torch.utils.data.random_split(dataset, [train_size, val_size])\n\nbatch_size = 4000\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)\n\ninput_size = x.shape[1]\noutput_size = y.shape[1]\nhidden_size = input_size + output_size\nmodel = FFNN(input_size, [3*hidden_size, 2*hidden_size, hidden_size, 2*hidden_size, 3*hidden_size], output_size).to(device)\ncriterion = nn.MSELoss()  # Using MSE for regression\noptimizer = optim.AdamW(model.parameters(), lr=0.001, weight_decay=0.01)\nscheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=3, verbose=False)","metadata":{"execution":{"iopub.status.busy":"2024-04-26T05:59:31.40986Z","iopub.execute_input":"2024-04-26T05:59:31.410224Z","iopub.status.idle":"2024-04-26T05:59:34.402081Z","shell.execute_reply.started":"2024-04-26T05:59:31.410197Z","shell.execute_reply":"2024-04-26T05:59:34.401125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training loop\nepochs = 100000\nbest_val_loss = float('inf')  # Set initial best as infinity\nbest_model_state = None       # To store the best model's state\npatience_count = 0\npatience = 5\nfor epoch in range(epochs):\n    model.train()\n    total_loss = 0\n    steps = 0\n    for batch_idx, (inputs, labels) in enumerate(train_loader):\n        optimizer.zero_grad()\n        outputs = model(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n\n        total_loss += loss.item()\n        steps += 1\n\n        # Print every 10 steps\n        if (batch_idx + 1) % 100 == 0:\n            print(f'Epoch {epoch + 1}, Step {batch_idx + 1}, Training Loss: {total_loss / steps:.4f}')\n            total_loss = 0  # Reset the loss for the next 10 steps\n            steps = 0  # Reset step count\n    \n\n    # Validation step\n    model.eval()\n    val_loss = 0\n    with torch.no_grad():\n        for inputs, labels in val_loader:\n            outputs = model(inputs)\n            val_loss += criterion(outputs, labels).item()\n\n    avg_val_loss = val_loss / len(val_loader)\n    print(f'Epoch {epoch+1}, Validation Loss: {avg_val_loss}')\n    \n    scheduler.step(avg_val_loss)  # Adjust learning rate\n\n    # Update best model if current epoch's validation loss is lower\n    if avg_val_loss < best_val_loss:\n        best_val_loss = avg_val_loss\n        best_model_state = model.state_dict()  # Save the best model state\n        patience_count = 0\n        print(\"Validation loss decreased, saving new best model and resetting patience counter.\")\n    else:\n        patience_count += 1\n        print(f\"No improvement in validation loss for {patience_count} epochs.\")\n        \n    if patience_count >= patience:\n        print(\"Stopping early due to no improvement in validation loss.\")\n        break","metadata":{"execution":{"iopub.status.busy":"2024-04-26T05:59:38.843902Z","iopub.execute_input":"2024-04-26T05:59:38.84444Z","iopub.status.idle":"2024-04-26T06:00:47.328856Z","shell.execute_reply.started":"2024-04-26T05:59:38.844407Z","shell.execute_reply":"2024-04-26T06:00:47.327779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test\nif not DEBUGGING:\n    model.load_state_dict(best_model_state)\n    model.eval()\n    predt = np.zeros([xt.shape[0], output_size], dtype=np.float32)  # output_size is the dimension of your model's output\n    batch_size = 1024 * 128  # Batch size for inference\n\n    i1 = 0\n    for i in range(10000):\n        i2 = np.minimum(i1 + batch_size, xt.shape[0])\n        if i1 == i2:  # Break the loop if range does not change\n            break\n\n        # Convert the current slice of xt to a PyTorch tensor\n        inputs = torch.from_numpy(xt[i1:i2, :]).float().to(device)\n\n        # No need to track gradients for inference\n        with torch.no_grad():\n            outputs = model(inputs)  # Get model predictions\n            predt[i1:i2, :] = outputs.cpu().numpy()  # Store predictions in predt\n\n        print(np.round(i2 / predt.shape[0], 2))  # Print the percentage completion\n        i1 = i2  # Update i1 to the end of the current batch\n\n        if i2 >= xt.shape[0]:\n            break","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not DEBUGGING:\n    # submit\n    # override constant columns\n    for i in range(sy.shape[0]):\n        if sy[i] < min_std * 1.1:\n            predt[:,i] = 0\n\n    # undo y scaling\n    predt = predt * sy.reshape(1,-1) + my.reshape(1,-1)\n\n    ss = pd.read_csv(\"/kaggle/input/leap-atmospheric-physics-ai-climsim/sample_submission.csv\")\n    ss.iloc[:,1:] *= predt\n    ss.to_csv(\"submission.csv\", index=False)","metadata":{},"execution_count":null,"outputs":[]}]}