{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":101849,"databundleVersionId":12846694,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# step 0\n\nimport numpy as np\nimport pandas as pd\nimport pyarrow.parquet as pq\nimport os, glob\nimport matplotlib.pyplot as plt\nfrom scipy.ndimage import median_filter\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.ensemble import RandomForestRegressor\nimport lightgbm as lgb","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T06:14:10.696732Z","iopub.execute_input":"2025-07-11T06:14:10.697056Z","iopub.status.idle":"2025-07-11T06:14:10.702137Z","shell.execute_reply.started":"2025-07-11T06:14:10.697031Z","shell.execute_reply":"2025-07-11T06:14:10.701186Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# step 1\n\ntrain_df = pd.read_csv('/kaggle/input/ariel-data-challenge-2025/train.csv')\nprint(\"Train shape:\", train_df.shape)\n\nwavelengths_df = pd.read_csv('/kaggle/input/ariel-data-challenge-2025/wavelengths.csv')\nprint(\"Wavelengths shape:\", wavelengths_df.shape)\n\nadc_info = pd.read_csv('/kaggle/input/ariel-data-challenge-2025/adc_info.csv')\nprint(adc_info)\n\ntrain_star_info = pd.read_csv('/kaggle/input/ariel-data-challenge-2025/train_star_info.csv')\nprint(\"Star info shape:\", train_star_info.shape)\n\naxis_info = pq.read_table('/kaggle/input/ariel-data-challenge-2025/axis_info.parquet').to_pandas()\nprint(\"Axis Info shape:\", axis_info.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T06:14:12.603024Z","iopub.execute_input":"2025-07-11T06:14:12.603311Z","iopub.status.idle":"2025-07-11T06:14:12.730898Z","shell.execute_reply.started":"2025-07-11T06:14:12.60329Z","shell.execute_reply":"2025-07-11T06:14:12.730194Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_fgs1_data(planet_id, adc_info):\n    print(f'\\n--- Processing Planet ID: {planet_id} ---')\n    base_path = f\"/kaggle/input/ariel-data-challenge-2025/train/{planet_id}\"\n    calib_path = f\"{base_path}/FGS1_calibration_0\"\n\n    # Load raw signal\n    fgs1_signal = pq.read_table(f\"{base_path}/FGS1_signal_0.parquet\").to_pandas()\n    print(f\"FGS1 raw signal Shape: {fgs1_signal.shape}\")\n\n    # Load calibration frames and convert to NumPy\n    fgs1_dark = pq.read_table(f\"{calib_path}/dark.parquet\").to_pandas().to_numpy()\n    fgs1_flat = pq.read_table(f\"{calib_path}/flat.parquet\").to_pandas().to_numpy()\n    fgs1_dead = pq.read_table(f\"{calib_path}/dead.parquet\").to_pandas().to_numpy()\n    fgs1_linear_corr = pq.read_table(f\"{calib_path}/linear_corr.parquet\").to_pandas()\n\n    print(f\"Shape of dark: {fgs1_dark.shape}\")\n    print(f\"Shape of flat: {fgs1_flat.shape}\")\n    print(f\"Shape of dead: {fgs1_dead.shape}\")\n    print(f\"Shape of linear_corr: {fgs1_linear_corr.shape}\")\n\n    # Reshape raw signal from (135000, 1024) → (135000, 32, 32)\n    fgs1_signal_np = fgs1_signal.to_numpy().reshape((135000, 32, 32)).astype(np.float32)\n\n    # Apply calibration\n    signal = fgs1_signal_np - fgs1_dark[np.newaxis, :, :]\n    signal = signal / fgs1_flat[np.newaxis, :, :]\n    signal[:, fgs1_dead == 1] = np.nan\n\n    # Apply ADC\n    gain = adc_info['FGS1_adc_gain'].iloc[0]\n    offset = adc_info['FGS1_adc_offset'].iloc[0]\n    signal = signal * gain + offset\n\n    return signal","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T06:28:05.468685Z","iopub.execute_input":"2025-07-11T06:28:05.46897Z","iopub.status.idle":"2025-07-11T06:28:05.47656Z","shell.execute_reply.started":"2025-07-11T06:28:05.468951Z","shell.execute_reply":"2025-07-11T06:28:05.475586Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_airs_data(planet_id, adc_info):\n    print(f'\\n--- Processing AIRS Data for Planet ID: {planet_id} ---')\n    base_path = f\"/kaggle/input/ariel-data-challenge-2025/train/{planet_id}\"\n    calib_path = f\"{base_path}/AIRS-CH0_calibration_0\"\n\n    # Load raw signal\n    airs_signal = pq.read_table(f\"{base_path}/AIRS-CH0_signal_0.parquet\").to_pandas()\n    print(f\"AIRS raw signal shape: {airs_signal.shape}\")\n\n    # Load calibration files and convert to NumPy\n    airs_dark = pq.read_table(f\"{calib_path}/dark.parquet\").to_pandas().to_numpy()\n    airs_flat = pq.read_table(f\"{calib_path}/flat.parquet\").to_pandas().to_numpy()\n    airs_dead = pq.read_table(f\"{calib_path}/dead.parquet\").to_pandas().to_numpy()\n    airs_linear_corr = pq.read_table(f\"{calib_path}/linear_corr.parquet\").to_pandas()\n\n    print(f\"Shape of dark: {airs_dark.shape}\")\n    print(f\"Shape of flat: {airs_flat.shape}\")\n    print(f\"Shape of dead: {airs_dead.shape}\")\n    print(f\"Shape of linear_corr: {airs_linear_corr.shape}\")\n\n    # Reshape signal from (11250, 11392) → (11250, 32, 356)\n    airs_signal_np = airs_signal.to_numpy().reshape((11250, 32, 356)).astype(np.float32)\n\n    # Calibration\n    signal = airs_signal_np - airs_dark[np.newaxis, :, :]\n    signal = signal / airs_flat[np.newaxis, :, :]\n    signal[:, airs_dead == 1] = np.nan\n\n    # Apply ADC\n    gain = adc_info['AIRS-CH0_adc_gain'].iloc[0]\n    offset = adc_info['AIRS-CH0_adc_offset'].iloc[0]\n    signal = signal * gain + offset\n\n    return signal","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T06:28:03.344464Z","iopub.execute_input":"2025-07-11T06:28:03.344818Z","iopub.status.idle":"2025-07-11T06:28:03.352798Z","shell.execute_reply.started":"2025-07-11T06:28:03.344797Z","shell.execute_reply":"2025-07-11T06:28:03.351925Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"planet_id = 1010375142\nfgs1_signal = process_fgs1_data(planet_id, adc_info)\nairs_signal = process_airs_data(planet_id, adc_info)\n\nprint(fgs1_signal)\nprint(\"-------------------------------------------\")\nprint(airs_signal)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-11T06:29:17.809655Z","iopub.execute_input":"2025-07-11T06:29:17.810746Z","iopub.status.idle":"2025-07-11T06:29:21.881644Z","shell.execute_reply.started":"2025-07-11T06:29:17.810706Z","shell.execute_reply":"2025-07-11T06:29:21.880872Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # step 2.2\n\n# def process_airs_data(planet_id, adc_info):\n#     print(f\"\\n--- Processing AIRS-CH0 for Planet ID: {planet_id} ---\")\n\n#     base_path = f\"/kaggle/input/ariel-data-challenge-2025/train/{planet_id}/\"\n#     calib_path = f\"{base_path}/AIRS-CH0_calibration_0/\"  # Fixed calibration path\n\n#     # 1. Load AIRS-CH0 signal (CORRECTED FILENAME)\n#     airs_ch0_signal = pq.read_table(f\"{base_path}/AIRS-CH0_signal_0.parquet\").to_pandas()\n#     print(f\"AIRS-CH0 raw signal shape: {airs_ch0_signal.shape}\")\n\n#     # 2. Load calibration data\n#     print(\"Loading calibration files...\")\n#     airs_dark_calib = pq.read_table(f\"{calib_path}/dark.parquet\").to_pandas()\n#     airs_flat_calib = pq.read_table(f\"{calib_path}/flat.parquet\").to_pandas()\n#     airs_dead_calib = pq.read_table(f\"{calib_path}/dead.parquet\").to_pandas()\n    \n#     print(f\"Dark calib shape: {airs_dark_calib.shape}\")  # Should be (32, 356)\n#     print(f\"Flat calib shape: {airs_flat_calib.shape}\")\n#     print(f\"Dead pixel map shape: {airs_dead_calib.shape}\")\n\n#     # 3. ADC correction\n#     try:\n#         airs_gain = adc_info.loc[adc_info['instrument'] == 'AIRS-CH0', 'gain'].values[0]\n#         airs_offset = adc_info.loc[adc_info['instrument'] == 'AIRS-CH0', 'offset'].values[0]\n#     except:\n#         # Fallback if column names are different\n#         airs_gain = adc_info['AIRS-CH0_adc_gain'].iloc[0]\n#         airs_offset = adc_info['AIRS-CH0_adc_offset'].iloc[0]\n        \n#     print(f\"AIRS ADC Gain: {airs_gain}\")\n#     print(f\"AIRS ADC Offset: {airs_offset}\")\n\n#     # 4. Apply ADC correction and RESHAPE PROPERLY\n#     airs_corrected = (airs_ch0_signal.values * airs_gain) + airs_offset\n#     print(f\"Corrected signal shape: {airs_corrected.shape}\")\n    \n#     # Correct reshape: AIRS has 11250 timepoints × (32 × 356) pixels\n#     try:\n#         airs_images = airs_corrected.reshape(-1, 32, 356)\n#     except ValueError:\n#         # Calculate expected number of pixels\n#         pixels = airs_corrected.shape[1]\n#         frames = airs_corrected.shape[0]\n#         print(f\"WARNING: {pixels} pixels, unable to reshape to (X,32,356). Using reshape({frames},32,356))\")\n#         airs_images = airs_corrected.reshape(frames, 32, 356)\n    \n#     print(f\"Reshaped AIRS images: {airs_images.shape}\")\n\n#     # 5. Calibration pipeline\n#     # Dark subtraction\n#     dark_frame = airs_dark_calib.values.squeeze()  # Handle possible extra dimensions\n#     if dark_frame.ndim == 3:\n#         dark_frame = np.median(dark_frame, axis=0)\n#     calibrated_images = airs_images.copy()\n#     calibrated_images -= dark_frame\n\n#     # Dead pixel correction\n#     dead_mask = (airs_dead_calib.values > 0).squeeze()  # Remove single-dimensional entries\n#     dead_mask = dead_mask.astype(bool)\n#     if dead_mask.shape != (32, 356):\n#         dead_mask = dead_mask.reshape(32, 356)\n#     print(f\"Dead pixels: {np.sum(dead_mask)}/{dead_mask.size}\")\n\n#     # Vectorized dead pixel fix\n#     from scipy.ndimage import generic_filter\n#     dead_mask_3d = np.zeros_like(calibrated_images, dtype=bool)\n#     dead_mask_3d[:] = dead_mask[np.newaxis, :, :]  # Broadcast mask to all frames\n    \n#     # Create filtered version\n#     filtered_airs = np.stack([\n#         generic_filter(calibrated_images[i], median_nan_fill, size=3, mode='mirror')\n#         for i in range(calibrated_images.shape[0])\n#     ])\n    \n#     calibrated_images[dead_mask_3d] = filtered_airs[dead_mask_3d]\n\n#     # Flat field correction\n#     flat_img = airs_flat_calib.values\n#     if flat_img.ndim == 3:\n#         flat_img = np.median(flat_img, axis=0)\n#     normalized_flat = flat_img / np.median(flat_img)\n#     calibrated_images /= normalized_flat\n\n#     # 6. Extract flux per wavelength channel (sum over rows)\n#     # This is CRITICAL for AIRS spectral separation\n#     print(\"Extracting flux per spectral column...\")\n#     flux_per_wavelength = np.sum(calibrated_images, axis=1)  # Sum over rows (32 → 1)\n#     print(f\"Flux per wavelength shape: {flux_per_wavelength.shape}\")  # (11250, 356)\n\n#     # 7. Visualization and return\n#     # Plot light curve for first spectral channel\n#     plt.figure(figsize=(12,4))\n#     plt.plot(flux_per_wavelength[:, 0])\n#     plt.title(f\"AIRS-CH0 Spectra Channel 0 - Planet {planet_id}\")\n    \n#     # Heatmap of first frame\n#     plt.figure(figsize=(10,6))\n#     plt.imshow(calibrated_images[0], aspect='auto', cmap='viridis')\n#     plt.title(\"First Calibrated Frame\")\n\n#     return flux_per_wavelength  # Return (11250, 356) spectral flux array\n\n# # Helper function for vectorized dead pixel replacement\n# def median_nan_fill(window):\n#     \"\"\"Replace center pixel with median of neighbors if NaN\"\"\"\n#     center = window[window.size//2]\n#     neighbors = np.delete(window, window.size//2)\n#     return np.nanmedian(neighbors) if np.isnan(center) else center\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-04T07:46:10.869462Z","iopub.execute_input":"2025-07-04T07:46:10.870503Z","iopub.status.idle":"2025-07-04T07:46:10.886711Z","shell.execute_reply.started":"2025-07-04T07:46:10.870471Z","shell.execute_reply":"2025-07-04T07:46:10.885456Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def extract_spectrum_from_flux(fgs1_flux, airs_flux):\n#     \"\"\"\n#     Reduce time series flux into 1D spectrum and corresponding uncertainty.\n#     Returns:\n#         - mu (length 283)\n#         - sigma (length 283)\n#     \"\"\"\n#     # FGS1 is a time series → reduce to 1 value (mean + std)\n#     mu_fgs1 = np.mean(fgs1_flux)\n#     sigma_fgs1 = np.std(fgs1_flux)\n\n#     # AIRS is (11250, 356) → mean over time gives shape (356,)\n#     mu_airs = np.mean(airs_flux, axis=0)[:282]\n#     sigma_airs = np.std(airs_flux, axis=0)[:282]\n\n#     # Combine into total 283\n#     mu = np.concatenate(([mu_fgs1], mu_airs))  # 1 + 282 = 283\n#     sigma = np.concatenate(([sigma_fgs1], sigma_airs))\n\n#     return mu, sigma","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-04T07:46:15.343955Z","iopub.execute_input":"2025-07-04T07:46:15.344329Z","iopub.status.idle":"2025-07-04T07:46:15.351414Z","shell.execute_reply.started":"2025-07-04T07:46:15.344301Z","shell.execute_reply":"2025-07-04T07:46:15.350216Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def format_submission_row(planet_id, mu, sigma):\n#     return [planet_id] + list(mu) + list(sigma)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-04T07:46:18.138963Z","iopub.execute_input":"2025-07-04T07:46:18.139403Z","iopub.status.idle":"2025-07-04T07:46:18.145782Z","shell.execute_reply.started":"2025-07-04T07:46:18.139371Z","shell.execute_reply":"2025-07-04T07:46:18.144467Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def process_single_planet(planet_id, adc_info):\n#     try:\n#         print(f\"\\n🚀 Starting processing for planet: {planet_id}\")\n\n#         fgs1_flux, _ = process_fgs1_data(planet_id, adc_info)\n#         airs_flux = process_airs_data(planet_id, adc_info)\n\n#         mu, sigma = extract_spectrum_from_flux(fgs1_flux, airs_flux)\n#         submission_row = format_submission_row(planet_id, mu, sigma)\n\n#         # Header format fix\n#         mu_cols = [f\"wl_{i+1}\" for i in range(283)]\n#         sigma_cols = [f\"sigma_{i+1}\" for i in range(283)]\n#         header = [\"planet_id\"] + mu_cols + sigma_cols\n\n#         submission_df = pd.DataFrame([submission_row], columns=header)\n#         submission_df.to_csv(\"submission.csv\", index=False)\n\n#         print(\"✅ Done! Created submission.csv for planet:\", planet_id)\n#         return submission_df\n\n#     except Exception as e:\n#         print(f\"❌ Failed for planet {planet_id}: {e}\")\n#         return None\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-04T07:46:22.549185Z","iopub.execute_input":"2025-07-04T07:46:22.54961Z","iopub.status.idle":"2025-07-04T07:46:22.559786Z","shell.execute_reply.started":"2025-07-04T07:46:22.549557Z","shell.execute_reply":"2025-07-04T07:46:22.558801Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# planet_id = 1010375142\n# process_single_planet(planet_id, adc_info)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-04T07:46:26.628765Z","iopub.execute_input":"2025-07-04T07:46:26.629153Z","iopub.status.idle":"2025-07-04T07:59:25.209711Z","shell.execute_reply.started":"2025-07-04T07:46:26.629117Z","shell.execute_reply":"2025-07-04T07:59:25.208666Z"}},"outputs":[],"execution_count":null}]}