{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":101849,"databundleVersionId":13093295,"sourceType":"competition"},{"sourceId":509283,"sourceType":"modelInstanceVersion","modelInstanceId":403977,"modelId":421901}],"dockerImageVersionId":31089,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn.functional as F\nimport multiprocessing as mp\nimport torch.nn as nn\nimport os\nimport matplotlib.pyplot as plt\nimport itertools\n\nfrom tqdm import tqdm\nfrom astropy.stats import sigma_clip\nfrom scipy.optimize import minimize\nfrom torch.utils.data import DataLoader, TensorDataset, random_split\nfrom sklearn.preprocessing import StandardScaler\nfrom scipy.signal import savgol_filter\nfrom astropy.stats import sigma_clip\n\nROOT_PATH = \"/kaggle/input/ariel-data-challenge-2025\"\nMODE = \"test\"\n\nclass Config:\n    CUT_INF = 39\n    CUT_SUP = 250\n\n    SENSOR_CONFIG = {\n        \"AIRS-CH0\": {\n            \"raw_shape\": [11250, 32, 356],\n            \"calibrated_shape\": [1, 32, CUT_SUP - CUT_INF],\n            \"linear_corr_shape\": (6, 32, 356),\n            \"dt_pattern\": (0.1, 4.5),\n            \"binning\": 30\n        },\n        \"FGS1\": {\n            \"raw_shape\": [135000, 32, 32],\n            \"calibrated_shape\": [1, 32, 32],\n            \"linear_corr_shape\": (6, 32, 32),\n            \"dt_pattern\": (0.1, 0.1),\n            \"binning\": 30 * 12\n        }\n    }\n\n\nclass SignalProcessor:\n    def __init__(self, config):\n        self.cfg = config\n        self.planet_ids = pd.read_csv(f\"{ROOT_PATH}/{MODE}_star_info.csv\", index_col='planet_id').index.astype(int)\n\n    def _calibrate_single_signal(self, planet_id, sensor):\n        cfg = self.cfg.SENSOR_CONFIG[sensor]\n        path = f\"{ROOT_PATH}/{MODE}/{planet_id}/\"\n\n        signal = pd.read_parquet(path + f\"{sensor}_signal_0.parquet\").to_numpy().reshape(cfg[\"raw_shape\"])\n        dark = pd.read_parquet(path + f\"{sensor}_calibration_0/dark.parquet\").to_numpy()\n        dead = pd.read_parquet(path + f\"{sensor}_calibration_0/dead.parquet\").to_numpy()\n        flat = pd.read_parquet(path + f\"{sensor}_calibration_0/flat.parquet\").to_numpy()\n        linear_corr = pd.read_parquet(path + f\"{sensor}_calibration_0/linear_corr.parquet\").to_numpy().astype(np.float64).reshape(cfg[\"linear_corr_shape\"])\n\n        gain, offset = 0.4369, -1000.0\n        signal = signal / gain + offset\n\n        hot = sigma_clip(dark, sigma=5, maxiters=5).mask\n\n        if sensor == \"AIRS-CH0\":\n            inf = self.cfg.CUT_INF\n            sup = self.cfg.CUT_SUP\n            signal = signal[:, :, inf:sup]\n            linear_corr = linear_corr[:, :, inf:sup]\n            dark = dark[:, inf:sup]\n            dead = dead[:, inf:sup]\n            flat = flat[:, inf:sup]\n            hot = hot[:, inf:sup]\n\n        base_dt, increment = cfg[\"dt_pattern\"]\n        dt = np.ones(len(signal)) * base_dt\n        dt[1::2] += increment\n\n        signal = signal.clip(0)\n        linear_corr_flipped = np.flip(linear_corr, axis=0)\n        corrected_signal = signal.copy()\n        \n        for x, y in itertools.product(range(signal.shape[1]), range(signal.shape[2])):\n            poly = np.poly1d(linear_corr_flipped[:, x, y])\n            corrected_signal[:, x, y] = poly(corrected_signal[:, x, y])\n        signal = corrected_signal\n        signal -= dark * dt[:, np.newaxis, np.newaxis]\n\n        flat = flat.reshape(cfg[\"calibrated_shape\"])\n        dead_mask = dead.reshape(cfg[\"calibrated_shape\"])\n        hot_mask = hot.reshape(cfg[\"calibrated_shape\"])\n\n        flat[dead_mask] = np.nan\n        flat[hot_mask] = np.nan\n\n        signal = signal / flat\n\n        return signal\n\n    def _preprocess_calibrated_signal(self, signal, sensor):\n        cfg = self.cfg.SENSOR_CONFIG[sensor]\n        binning = cfg[\"binning\"]\n\n        if sensor == \"AIRS-CH0\":\n            roi = signal[:, 10:22, :]\n        else:\n            roi = signal[:, 10:22, 10:22]\n            roi = roi.reshape(roi.shape[0], -1)\n\n        mean_signal = np.nanmean(roi, axis=1)\n        \n        cds_signal = mean_signal[1::2] - mean_signal[0::2]\n\n        n_bins = cds_signal.shape[0] // binning\n        binned = np.array([\n            cds_signal[j*binning : (j+1)*binning].mean(axis=0) \n            for j in range(n_bins)\n        ])\n\n        if sensor == \"FGS1\":\n            binned = binned.reshape((binned.shape[0], 1))\n        return binned\n\n\n    def process_all_data(self):\n        combined_signals = []\n    \n        for planet_id in tqdm(self.planet_ids):\n            signal_fgs = self._calibrate_single_signal(planet_id, \"FGS1\")\n            processed_fgs = self._preprocess_calibrated_signal(signal_fgs, \"FGS1\")\n    \n            signal_airs = self._calibrate_single_signal(planet_id, \"AIRS-CH0\")\n            processed_airs = self._preprocess_calibrated_signal(signal_airs, \"AIRS-CH0\")\n    \n            combined = np.concatenate([processed_fgs, processed_airs], axis=1)\n            combined_signals.append(combined)\n    \n        result = np.stack(combined_signals)\n        return result\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-08-10T13:56:23.309977Z","iopub.execute_input":"2025-08-10T13:56:23.310488Z","iopub.status.idle":"2025-08-10T13:56:28.324566Z","shell.execute_reply.started":"2025-08-10T13:56:23.310456Z","shell.execute_reply":"2025-08-10T13:56:28.324019Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"processor = SignalProcessor(Config)\nStarInfo = pd.read_csv(ROOT_PATH + f\"/{MODE}_star_info.csv\")\nStarInfo[\"planet_id\"] = StarInfo[\"planet_id\"].astype(int)\nPlanetIds = StarInfo[\"planet_id\"].tolist()\nStarInfo = StarInfo.set_index(\"planet_id\")\nsignal_train = processor.process_all_data()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-10T13:56:28.325667Z","iopub.execute_input":"2025-08-10T13:56:28.32604Z","iopub.status.idle":"2025-08-10T13:56:49.968519Z","shell.execute_reply.started":"2025-08-10T13:56:28.32602Z","shell.execute_reply":"2025-08-10T13:56:49.967834Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def phase_detector(signal):\n    min_index = np.argmin(signal[30:140]) + 30\n    signal1 = signal[:min_index]\n    grad1 = np.gradient(signal1)\n    grad1 /= grad1.max()\n    signal2 = signal[min_index:]\n    grad2 = np.gradient(signal2)\n    grad2 /= grad2.max()\n\n    phase1 = np.argmin(grad1)\n    phase2 = np.argmax(grad2) + min_index\n\n    return phase1, phase2\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-10T13:56:49.969236Z","iopub.execute_input":"2025-08-10T13:56:49.969516Z","iopub.status.idle":"2025-08-10T13:56:49.974221Z","shell.execute_reply.started":"2025-08-10T13:56:49.969491Z","shell.execute_reply":"2025-08-10T13:56:49.973631Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def compute_s_opt(signal, phase1, phase2, buffer_size = 7, degree = 3):\n    delta = buffer_size\n    if phase1 - delta <= 0 or phase2 + delta >= len(signal) or phase2 - delta - (phase1 + delta) < 5:\n        delta = 2\n    def objective(s):\n        middle = signal[phase1 + delta : phase2 - delta] * (1 + s)\n        y = np.concatenate([signal[:phase1 - delta], middle, signal[phase2 + delta:]])\n        x = np.arange(len(y))\n        coeffs = np.polyfit(x, y, degree)\n        poly = np.poly1d(coeffs)        \n        return np.abs(poly(x) - y).mean()\n\n    res = minimize(objective, 0.0001, method=\"Nelder-Mead\")\n    return res.x[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-10T13:56:49.975842Z","iopub.execute_input":"2025-08-10T13:56:49.976079Z","iopub.status.idle":"2025-08-10T13:56:49.986047Z","shell.execute_reply.started":"2025-08-10T13:56:49.976062Z","shell.execute_reply":"2025-08-10T13:56:49.985388Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_shifts = []\nfor IDX in tqdm(range(len(signal_train))):\n    data = signal_train[IDX]\n    normalized_data = data[:, 1:].mean(1)\n    normalized_data = savgol_filter(normalized_data, 30, 2)\n    phase1, phase2 = phase_detector(normalized_data)\n    buffer_size_poly = 7\n    phase1 = max(7, phase1)\n    phase2 = min(len(normalized_data) - 7 - 1, phase2) \n    opt = compute_s_opt(normalized_data, phase1, phase2, buffer_size_poly, degree = 3)\n    y_shifts.append(opt)\ny_shifts = np.array(y_shifts) * 0.93960\ndf = pd.DataFrame({\"transit_depth\":y_shifts},index=StarInfo.index,)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-10T13:56:49.986712Z","iopub.execute_input":"2025-08-10T13:56:49.986976Z","iopub.status.idle":"2025-08-10T13:56:50.06733Z","shell.execute_reply.started":"2025-08-10T13:56:49.986958Z","shell.execute_reply":"2025-08-10T13:56:50.0667Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"input_df = pd.merge(df, StarInfo, on=\"planet_id\", how=\"left\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-10T13:56:50.067953Z","iopub.execute_input":"2025-08-10T13:56:50.0682Z","iopub.status.idle":"2025-08-10T13:56:50.076991Z","shell.execute_reply.started":"2025-08-10T13:56:50.068178Z","shell.execute_reply":"2025-08-10T13:56:50.076383Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"input_df[\"transit_depth\"] *= 10000\nfeatures = ['transit_depth','Rs','i']\nX = input_df[features].values.astype(np.float32)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-10T13:56:50.077733Z","iopub.execute_input":"2025-08-10T13:56:50.077984Z","iopub.status.idle":"2025-08-10T13:56:50.089583Z","shell.execute_reply.started":"2025-08-10T13:56:50.077966Z","shell.execute_reply":"2025-08-10T13:56:50.089047Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nclass ResidualBlock(nn.Module):\n    def __init__(self, dim, p=0.2):\n        super().__init__()\n        self.fc1 = nn.Linear(dim, dim)\n        self.bn1 = nn.BatchNorm1d(dim)\n        self.fc2 = nn.Linear(dim, dim)\n        self.bn2 = nn.BatchNorm1d(dim)\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(p)\n\n    def forward(self, x):\n        identity = x\n        out = self.relu(self.bn1(self.fc1(x)))\n        out = self.dropout(out)\n        out = self.bn2(self.fc2(out))\n        return self.relu(out + identity)\n\n\nclass ResNetMLP(nn.Module):\n    def __init__(self, input_dim=3, hidden_dim=32, output_dim=1, num_blocks=3, dropout_rate=0.2):\n        super().__init__()\n        self.input_layer = nn.Linear(input_dim, hidden_dim)\n        self.blocks = nn.Sequential(*[ResidualBlock(hidden_dim, p=dropout_rate) for _ in range(num_blocks)])\n        self.output_layer = nn.Linear(hidden_dim, output_dim)\n\n    def forward(self, x):\n        x = self.input_layer(x)\n        x = self.blocks(x)\n        x = self.output_layer(x)\n        return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-10T13:56:50.090446Z","iopub.execute_input":"2025-08-10T13:56:50.09071Z","iopub.status.idle":"2025-08-10T13:56:50.100428Z","shell.execute_reply.started":"2025-08-10T13:56:50.090684Z","shell.execute_reply":"2025-08-10T13:56:50.099753Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = ResNetMLP(num_blocks=80, dropout_rate=0.5)\nmodel.load_state_dict(torch.load(\"/kaggle/input/net/pytorch/default/1/best_artifacts/best_model.pth\"))\nmodel.eval()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-10T13:56:50.101085Z","iopub.execute_input":"2025-08-10T13:56:50.10127Z","iopub.status.idle":"2025-08-10T13:56:50.441315Z","shell.execute_reply.started":"2025-08-10T13:56:50.101256Z","shell.execute_reply":"2025-08-10T13:56:50.44058Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_tensor = torch.tensor(X, dtype=torch.float32)\nwith torch.no_grad():\n    y_pred = model(X_tensor).numpy()\ny_pred /= 10000","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-10T13:56:50.44318Z","iopub.execute_input":"2025-08-10T13:56:50.443576Z","iopub.status.idle":"2025-08-10T13:56:50.482188Z","shell.execute_reply.started":"2025-08-10T13:56:50.443559Z","shell.execute_reply":"2025-08-10T13:56:50.481567Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample = pd.read_csv(\"/kaggle/input/ariel-data-challenge-2025/sample_submission.csv\")\nsample_columns = sample.columns.drop(\"planet_id\")\nn_outputs = len(sample_columns) // 2\nrepeated_predictions = np.repeat(df.values, n_outputs).reshape(len(df), -1)\nrepeated_predictions[:,0] = (y_pred[:,0] * 0.8 + (X[:,0] * 0.2 )/ 10000)\nrepeated_predictions = repeated_predictions.clip(0)\nsigmas = np.ones_like(repeated_predictions)\nsubmission_values = np.concatenate([repeated_predictions, sigmas], axis=1)\nsubmission_df = pd.DataFrame(submission_values, columns=sample_columns, index=df.index)\nsubmission_df.insert(0, \"planet_id\", df.index)\nsubmission_df.to_csv(\"submission.csv\", index=False)\npd.read_csv(\"submission.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-10T13:56:50.482751Z","iopub.execute_input":"2025-08-10T13:56:50.482941Z","iopub.status.idle":"2025-08-10T13:56:50.531835Z","shell.execute_reply.started":"2025-08-10T13:56:50.48292Z","shell.execute_reply":"2025-08-10T13:56:50.531316Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}