{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":101849,"databundleVersionId":12846694,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🚀 ARIEL DATA CHALLENGE 2025-  EDA","metadata":{}},{"cell_type":"code","source":"# ====================================================================\n# 📚 IMPORTS AND SETUP\n# ====================================================================\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom pathlib import Path\nfrom scipy.signal import medfilt\nfrom scipy import stats\nfrom IPython.display import display, Markdown\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-20T17:00:49.13447Z","iopub.execute_input":"2025-07-20T17:00:49.134843Z","iopub.status.idle":"2025-07-20T17:00:50.818482Z","shell.execute_reply.started":"2025-07-20T17:00:49.134816Z","shell.execute_reply":"2025-07-20T17:00:50.817072Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ====================================================================\n# ⚙️ Configuration\n# ====================================================================\nBASE_PATH = Path('/kaggle/input/ariel-data-challenge-2025/')\nSAMPLE_PLANET_ID = '1010375142'\n\n# Styling\nplt.style.use('seaborn-v0_8-whitegrid')\nsns.set_context(\"talk\")\nplt.rcParams['figure.dpi'] = 100\nplt.rcParams['savefig.dpi'] = 150\n\ndef display_header(title):\n    \"\"\"Helper function to display styled headers.\"\"\"\n    display(Markdown(f\"### {title}\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-20T17:00:50.8203Z","iopub.execute_input":"2025-07-20T17:00:50.820719Z","iopub.status.idle":"2025-07-20T17:00:50.827824Z","shell.execute_reply.started":"2025-07-20T17:00:50.820694Z","shell.execute_reply":"2025-07-20T17:00:50.826558Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 🎯 SECTION 1: COMPETITION OVERVIEW & CORE TASK","metadata":{}},{"cell_type":"code","source":"# ====================================================================\n# 🎯 SECTION 1: COMPETITION OVERVIEW & CORE TASK\n# ====================================================================\n\ndisplay_header(\"🎯 Competition Overview\")\ndisplay(Markdown(\n    \"**The Ariel Data Challenge 2025** is a ML competition focused on extracting \"\n    \"exoplanet atmospheric spectra from noisy telescope observations. This is a **supervised \"\n    \"denoising problem** where we can predict 283 spectral values and their uncertainties \"\n    \"from complex time-series data.\"\n))\n\nprint(\"🔍 CORE CHALLENGE:\")\nprint(\"• Extract faint exoplanet atmospheric signals from noisy telescope data\")\nprint(\"• Extract 283 spectral values + uncertainties from complex 3D time-series data\")\nprint(\"• Handle complex 3D data cubes (time × spatial × spectral dimensions)\")\nprint(\"• Predict both mean spectrum values AND uncertainty estimates\")\nprint(\"• Score optimization using Gaussian Log-Likelihood (GLL) metric\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-20T17:00:50.829017Z","iopub.execute_input":"2025-07-20T17:00:50.829384Z","iopub.status.idle":"2025-07-20T17:00:50.865811Z","shell.execute_reply.started":"2025-07-20T17:00:50.829351Z","shell.execute_reply":"2025-07-20T17:00:50.864724Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 🎯 SECTION 2: METADATA DEEP DIVE","metadata":{}},{"cell_type":"code","source":"# ====================================================================\n# 🎯 SECTION 2: METADATA DEEP DIVE\n# ====================================================================\n\n# --- 2.1 The Ground Truth: train.csv ---\ndisplay_header(\"`train.csv`: The Ground Truth (Target)\")\ntrain_df = pd.read_csv(BASE_PATH / 'train.csv')\n\nprint(f\"📊 Training Data Shape: {train_df.shape}\")\nprint(f\"📊 Sample Planet ID: {SAMPLE_PLANET_ID}\")\n\n# Sample data\nsample_data = train_df[train_df['planet_id'] == int(SAMPLE_PLANET_ID)]\ndisplay(sample_data)\n\ndisplay(Markdown(\n    \"**💡 Significance:** This is the **target variable**. Each row represents one planet, \"\n    \"with 283 `wl_*` columns containing the true, clean atmospheric spectrum values. \"\n    \"The model must learn to predict these values from the raw observational data.\"\n))\n\n# --- 2.2 The Spectral Grid: wavelengths.csv ---\ndisplay_header(\"`wavelengths.csv`: The Spectral Grid\")\nwavelengths_df = pd.read_csv(BASE_PATH / 'wavelengths.csv')\n\nprint(f\"📊 Wavelength Grid Shape: {wavelengths_df.shape}\")\ndisplay(wavelengths_df.iloc[:, :5])  # first 5 wavelengths\n\n# Wavelength array\nwavelengths = wavelengths_df.iloc[0].values\nprint(f\"📊 Wavelength Statistics:\")\nprint(f\"   • Range: {wavelengths.min():.3f} - {wavelengths.max():.3f} μm\")\nprint(f\"   • Mean spacing: {np.diff(wavelengths).mean():.6f} μm\")\nprint(f\"   • Total points: {len(wavelengths)}\")\n\ndisplay(Markdown(\n    \"**💡 Significance:** This wavelengths.csv file provides the **physical wavelength mapping** for each \"\n    \"spectral point. Essential for physics-informed feature engineering and instrument-specific \"\n    \"processing (FGS1 vs AIRS-CH0 ranges).\"\n))\n\n# --- 2.3 The Time Axis: axis_info.parquet ---\ndisplay_header(\"`axis_info.parquet`: The Time Axis\")\naxis_info_df = pd.read_parquet(BASE_PATH / 'axis_info.parquet')\n\nprint(f\"📊 Axis Info Shape: {axis_info_df.shape}\")\nprint(f\"📊 Column Names: {axis_info_df.columns.tolist()}\")\nprint(f\"📊 Data Types: {axis_info_df.dtypes.to_dict()}\")\n\ndisplay(axis_info_df.head())\n\nprint(\"\\n🔍 TIME AXIS INSPECTION:\")\nfor col in axis_info_df.columns:\n    col_data = axis_info_df[col]\n    print(f\"   • {col}:\")\n    print(f\"     - Range: {col_data.min():.6f} to {col_data.max():.6f}\")\n    print(f\"     - Mean: {col_data.mean():.6f}\")\n    print(f\"     - Non-null count: {col_data.notna().sum()}\")\n\ndisplay(Markdown(\n    \"**💡 Significance:** This axis_info.parquet defines the **temporal dimension** of the raw data. \"\n    \"The structure shows separate time axes for different instruments and observation modes. \"\n    \"Critical for time-series analysis and transit event detection.\"\n))\n\n# --- 2.4 The Conversion Key: adc_info.csv ---\ndisplay_header(\"`adc_info.csv`: The ADC Conversion Key\")\nadc_info_df = pd.read_csv(BASE_PATH / 'adc_info.csv')\n\nprint(f\"📊 ADC Info Shape: {adc_info_df.shape}\")\ndisplay(adc_info_df)\n\n# Conversion parameters\nfgs1_gain = adc_info_df['FGS1_adc_gain'].iloc[0]\nfgs1_offset = adc_info_df['FGS1_adc_offset'].iloc[0]\nairs_gain = adc_info_df['AIRS-CH0_adc_gain'].iloc[0]\nairs_offset = adc_info_df['AIRS-CH0_adc_offset'].iloc[0]\n\nprint(f\"📊 Conversion Parameters:\")\nprint(f\"   • FGS1: gain={fgs1_gain:.6f}, offset={fgs1_offset:.6f}\")\nprint(f\"   • AIRS-CH0: gain={airs_gain:.6f}, offset={airs_offset:.6f}\")\n\ndisplay(Markdown(\n    \"**💡 Significance:** Raw observational data is stored as integers to save space. \"\n    \"These parameters convert to physical flux units using: \"\n    \"`physical_value = (raw_value × gain) + offset`. \"\n))\n\n# --- 2.5 Physical Context: train_star_info.csv ---\ndisplay_header(\"`train_star_info.csv`: Physical Context for Features\")\ntrain_star_info_df = pd.read_csv(BASE_PATH / 'train_star_info.csv')\n\nprint(f\"📊 Star Info Shape: {train_star_info_df.shape}\")\nsample_star_info = train_star_info_df[train_star_info_df['planet_id'] == int(SAMPLE_PLANET_ID)]\ndisplay(sample_star_info)\n\n# Statistical overview of stellar parameters\nprint(f\"📊 Stellar Parameter Statistics:\")\nnumeric_cols = train_star_info_df.select_dtypes(include=[np.number]).columns\nfor col in numeric_cols:\n    if col != 'planet_id':\n        values = train_star_info_df[col]\n        print(f\"   • {col}: {values.min():.3f} - {values.max():.3f} (mean: {values.mean():.3f})\")\n\ndisplay(Markdown(\n    \"**💡 Significance:** **Metadata** containing stellar and planetary parameters. \"\n    \"Properties like star temperature (`Ts`), planetary mass (`Mp`), and orbital inclination (`i`) \"\n    \"may influence transit signals. **Powerful features for model enhancement.**\"\n))\n\n# --- 2.6 Sample_submission.csv ---\ndisplay_header(\"`sample_submission.csv`: The Final Goal\")\nsample_submission_df = pd.read_csv(BASE_PATH / 'sample_submission.csv')\n\nprint(f\"📊 Submission Format Shape: {sample_submission_df.shape}\")\nprint(f\"📊 Expected Columns: {sample_submission_df.shape[1]} (1 + 283 + 283)\")\nprint(f\"📊 Test Planets: {len(sample_submission_df)}\")\n\ndisplay(sample_submission_df.head())\n\n# Submission format\nexpected_cols = 1 + 283 + 283  # planet_id + spectrum + uncertainty\nactual_cols = sample_submission_df.shape[1]\nformat_check = \"✅ CORRECT\" if actual_cols == expected_cols else \"❌ MISMATCH\"\n\nprint(f\"📊 Format Validation: {format_check}\")\nprint(f\"   • Expected: {expected_cols} columns\")\nprint(f\"   • Actual: {actual_cols} columns\")\n\ndisplay(Markdown(\n    \"**💡 Significance:** This defines the **exact submission format**. For each test planet (only one is there though), \"\n    \"we must predict 283 mean spectrum values (`wl_*`) AND 283 uncertainty values (`sigma_*`). \"\n))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-20T17:00:50.868178Z","iopub.execute_input":"2025-07-20T17:00:50.868483Z","iopub.status.idle":"2025-07-20T17:00:51.55135Z","shell.execute_reply.started":"2025-07-20T17:00:50.868454Z","shell.execute_reply":"2025-07-20T17:00:51.550172Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 🔧 SECTION 3: SIGNAL PROCESSING FUNCTIONS","metadata":{}},{"cell_type":"code","source":"# ====================================================================\n# 🔧 SECTION 3: SIGNAL PROCESSING FUNCTIONS\n# ====================================================================\n\ndisplay_header(\"🔧 Signal Processing Functions\")\n\ndef adc_calibration(raw_df, gain, offset):\n    \"\"\"Convert raw ADC uint16 values to physical flux units\"\"\"\n    return raw_df * gain + offset\n\ndef differential_reading(signal_df):\n    \"\"\"Extract differential signal using up-the-ramp sampling\"\"\"\n    # Extract even (start) and odd (end) frames\n    start_frames = signal_df.iloc[0::2].reset_index(drop=True)\n    end_frames = signal_df.iloc[1::2].reset_index(drop=True)\n    \n    # differential signal\n    diff_signal = end_frames - start_frames\n    return diff_signal\n\ndef create_light_curve(diff_signal):\n    \"\"\"Generate light curve by summing across all pixels\"\"\"\n    return diff_signal.sum(axis=1)\n\ndef sigma_clip(data, window_size=51, sigma=5):\n    \"\"\" sigma clipping for cosmic ray removal\"\"\"\n    local_median = medfilt(data, kernel_size=window_size)\n    residual = data - local_median\n    mad = np.median(np.abs(residual))\n    robust_std = mad * 1.4826\n    outliers = np.abs(residual) > (sigma * robust_std)\n    clean_data = data.copy()\n    clean_data[outliers] = np.nan\n    return clean_data, outliers\n\ndef polynomial_detrend(data, order=2, transit_mask=None):\n    \"\"\"Remove systematic trends using polynomial fitting\"\"\"\n    x_axis = np.arange(len(data))\n    \n    if transit_mask is not None:\n        fit_mask = ~transit_mask & ~np.isnan(data)\n    else:\n        fit_mask = ~np.isnan(data)\n    \n    poly_coeffs = np.polyfit(x_axis[fit_mask], data[fit_mask], order)\n    baseline_model = np.polyval(poly_coeffs, x_axis)\n    detrended = data - baseline_model\n    \n    return detrended, baseline_model\n\ndef detect_transit(light_curve, buffer_size=500):\n    \"\"\"Detect transit event in light curve\"\"\"\n    min_idx = np.nanargmin(light_curve)\n    min_depth = light_curve[min_idx]\n    \n    transit_start = max(0, min_idx - buffer_size)\n    transit_end = min(len(light_curve) - 1, min_idx + buffer_size)\n    \n    return transit_start, transit_end, min_depth\n\nprint(\"✅ Signal processing functions loaded successfully\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-20T17:00:51.552353Z","iopub.execute_input":"2025-07-20T17:00:51.552653Z","iopub.status.idle":"2025-07-20T17:00:51.568931Z","shell.execute_reply.started":"2025-07-20T17:00:51.552631Z","shell.execute_reply":"2025-07-20T17:00:51.567745Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 📊 SECTION 4: MERGE TRAINING DATA","metadata":{}},{"cell_type":"code","source":"# ====================================================================\n# 📊 SECTION 4: MERGE TRAINING DATA\n# ====================================================================\n\ndisplay_header(\"📊 Merge training data for comprehensive analysis\")\n\nmerged_train_df = pd.merge(train_df, train_star_info_df, on='planet_id')\n\n# Reshape wavelengths\nwavelengths_long_df = wavelengths_df.T.reset_index()\nwavelengths_long_df.columns = ['wl_id', 'wavelength_um']\nwavelengths_long_df['instrument'] = np.where(wavelengths_long_df['wavelength_um'] < 1.0, 'FGS1', 'AIRS-CH0')\n\n# Extract key parameters\nwavelengths = wavelengths_df.iloc[0].values\ngain = adc_info_df['FGS1_adc_gain'].iloc[0]\noffset = adc_info_df['FGS1_adc_offset'].iloc[0]\n\nprint(f\"📊 Dataset Overview:\")\nprint(f\"   • Training planets: {len(train_df):,}\")\nprint(f\"   • Test planets: {len(sample_submission_df):,}\")\nprint(f\"   • Wavelength points: {len(wavelengths)}\")\nprint(f\"   • Wavelength range: {wavelengths.min():.3f} - {wavelengths.max():.3f} μm\")\nprint(f\"   • ADC parameters: gain={gain:.4f}, offset={offset:.1f}\")\n    \n# Instrument analysis\nfgs1_count = (wavelengths_long_df['instrument'] == 'FGS1').sum()\nairs_count = (wavelengths_long_df['instrument'] == 'AIRS-CH0').sum()\n\nprint(f\"   • FGS1 points: {fgs1_count}\")\nprint(f\"   • AIRS-CH0 points: {airs_count}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-20T17:00:51.569883Z","iopub.execute_input":"2025-07-20T17:00:51.570153Z","iopub.status.idle":"2025-07-20T17:00:51.615642Z","shell.execute_reply.started":"2025-07-20T17:00:51.570125Z","shell.execute_reply":"2025-07-20T17:00:51.614635Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 🔬 SECTION 5: OBSERVATIONAL DATA EXPLORATION","metadata":{}},{"cell_type":"code","source":"# ====================================================================\n# 🔬 SECTION 5: OBSERVATIONAL DATA EXPLORATION\n# ====================================================================\n\n# --- 5.1 Instrument & Observation Set Analysis ---\ndisplay_header(\"🛰️ Instrument & Observation Set Analysis\")\n\nplanet_folders = [p for p in (BASE_PATH / 'train').iterdir() if p.is_dir()]\nprint(f\"📊 Total Planet Folders: {len(planet_folders)}\")\n\nsample_size = max(100, len(planet_folders)) # Total 1100 are there or sample 100 (by changing min or max)\nsample_folders = planet_folders[:sample_size]\n\n# Counting structure\nobservation_analysis = {\n    'single_observation_set': 0,      # Planets with _0 files only\n    'double_observation_set': 0,      # Planets with _0 and _1 files\n}\n\n# Planet information storage\nplanet_details = {}\n\nprint(f\"🔍 Analyzing observation sets for {sample_size} planets...\")\n\nfor planet_folder in sample_folders:\n    planet_id = planet_folder.name\n    \n    # Find all calibration folders\n    calibration_folders = [f for f in planet_folder.iterdir() \n                          if f.is_dir() and ('calibration' in f.name.lower())]\n    \n    # Find all signal files\n    signal_files = [f for f in planet_folder.iterdir() \n                   if f.is_file() and f.name.endswith('.parquet') and 'signal' in f.name]\n    \n    # Extract observation set numbers from calibration folders\n    calib_suffixes = set()\n    for calib_folder in calibration_folders:\n        # Extracting suffix from names like \"AIRS-CH0_calibration_0\" or \"FGS1_calibration_1\"\n        suffix = calib_folder.name.split('_')[-1]\n        calib_suffixes.add(suffix)\n    \n    # Extract observation set numbers from signal files\n    signal_suffixes = set()\n    for signal_file in signal_files:\n        # Extracting suffix from names like \"FGS1_signal_0.parquet\" or \"AIRS-CH0_signal_1.parquet\"\n        parts = signal_file.name.split('_')\n        if len(parts) >= 3:\n            suffix = parts[-1].replace('.parquet', '')\n            signal_suffixes.add(suffix)\n    \n    # Find common observation sets (both calibration and signal data available)\n    common_sets = calib_suffixes.intersection(signal_suffixes)\n    num_observation_sets = len(common_sets)\n    \n    # Store detailed information\n    planet_details[planet_id] = {\n        'observation_sets': sorted(list(common_sets)),\n        'num_sets': num_observation_sets,\n        'calibration_folders': len(calibration_folders),\n        'signal_files': len(signal_files),\n        'all_folders': [f.name for f in planet_folder.iterdir() if f.is_dir()],\n        'all_files': [f.name for f in planet_folder.iterdir() if f.is_file()]\n    }\n    \n    # Count observation sets\n    if num_observation_sets == 1:\n        observation_analysis['single_observation_set'] += 1\n    elif num_observation_sets == 2:\n        observation_analysis['double_observation_set'] += 1\n\nprint(f\"\\n📊 OBSERVATION SET ANALYSIS (sample of {sample_size} planets):\")\nprint(f\"   • Single observation set (e.g., _0 only): {observation_analysis['single_observation_set']} planets ({observation_analysis['single_observation_set']/sample_size*100:.1f}%)\")\nprint(f\"   • Double observation sets (e.g., _0 + _1): {observation_analysis['double_observation_set']} planets ({observation_analysis['double_observation_set']/sample_size*100:.1f}%)\")\n\n# Examples of different observation set patterns\nprint(f\"\\n🔍 EXAMPLE OBSERVATION PATTERNS:\")\nsingle_example = next((pid for pid, details in planet_details.items() if details['num_sets'] == 1), None)\ndouble_example = next((pid for pid, details in planet_details.items() if details['num_sets'] == 2), None)\n\nif single_example:\n    details = planet_details[single_example]\n    print(f\"   • Single set example (Planet {single_example}):\")\n    print(f\"     - Observation sets: {details['observation_sets']}\")\n    print(f\"     - Calibration folders: {details['calibration_folders']}\")\n    print(f\"     - Signal files: {details['signal_files']}\")\n    print(f\"     - All folders: {details['all_folders']}\")\n\nif double_example:\n    details = planet_details[double_example]\n    print(f\"   • Double set example (Planet {double_example}):\")\n    print(f\"     - Observation sets: {details['observation_sets']}\")\n    print(f\"     - Calibration folders: {details['calibration_folders']}\")\n    print(f\"     - Signal files: {details['signal_files']}\")\n    print(f\"     - All folders: {details['all_folders']}\")\n\n# First few planet details for debugging\nprint(f\"\\n🔍 FIRST FEW PLANET DETAILS (for debugging):\")\nfor i, (pid, details) in enumerate(list(planet_details.items())[:3]):\n    print(f\"   Planet {pid}:\")\n    print(f\"     - Folders: {details['all_folders']}\")\n    print(f\"     - Files: {details['all_files']}\")\n    print(f\"     - Observation sets found: {details['observation_sets']}\")\n\ndisplay(Markdown(\n    \"**💡 Insights:**\\n\\n\"\n    \"- **Multiple observation sets** may increase data volume and complexity\\n\"\n    \"- **Double observation planets** provide more training data but require careful handling\\n\"\n    \"- **Memory estimation** needs to factor in observation set multiplicity\\n\\n\"\n))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-20T17:00:51.617175Z","iopub.execute_input":"2025-07-20T17:00:51.617504Z","iopub.status.idle":"2025-07-20T17:00:59.377392Z","shell.execute_reply.started":"2025-07-20T17:00:51.617471Z","shell.execute_reply":"2025-07-20T17:00:59.376262Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 5.2 Visualizing a Flattened Image ---\ndisplay_header(f\"🖼️ Telescope Image Visualization (Planet {SAMPLE_PLANET_ID})\")\n\nfgs1_signal = None\ntry:\n    fgs1_signal = pd.read_parquet(BASE_PATH / f'train/{SAMPLE_PLANET_ID}/FGS1_signal_0.parquet')\n    print(f\"📊 FGS1 Signal Shape: {fgs1_signal.shape}\")\n    print(f\"📊 Time Steps: {len(fgs1_signal):,}\")\n    print(f\"📊 Pixels per Frame: {fgs1_signal.shape[1]}\")\n    \n    # Take a frame from the middle of the observation\n    frame_idx = len(fgs1_signal) // 2\n    flattened_image = fgs1_signal.iloc[frame_idx].values\n    \n    # Reshape from 1D vector (1024,) to 2D image (32, 32)\n    image_2d = flattened_image.reshape(32, 32)\n    \n    # Plot\n    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 6))\n    \n    # Plot 1: Raw image\n    im1 = ax1.imshow(image_2d, cmap='viridis', aspect='equal')\n    ax1.set_title(f'FGS1 Detector Image\\n(Planet {SAMPLE_PLANET_ID}, Frame {frame_idx:,})')\n    ax1.set_xlabel('Pixel Column')\n    ax1.set_ylabel('Pixel Row')\n    fig.colorbar(im1, ax=ax1, label='Flux (Raw ADC Units)')\n    \n    # Plot 2: Cross-section showing the stellar trace\n    middle_row = image_2d[16, :]  # Middle row\n    ax2.plot(range(32), middle_row, 'b-', linewidth=2, label='Stellar Trace')\n    ax2.set_title('Stellar Trace (Middle Row)')\n    ax2.set_xlabel('Pixel Column')\n    ax2.set_ylabel('Flux (Raw ADC Units)')\n    ax2.grid(True, alpha=0.3)\n    ax2.legend()\n    \n    plt.tight_layout()\n    plt.show()\n    \n    print(f\"📊 Image Statistics:\")\n    print(f\"   • Min flux: {flattened_image.min():.1f}\")\n    print(f\"   • Max flux: {flattened_image.max():.1f}\")\n    print(f\"   • Mean flux: {flattened_image.mean():.1f}\")\n    print(f\"   • Dynamic range: {flattened_image.max()/flattened_image.min():.2f}x\")\n    \n    display(Markdown(\n        \"**💡 Significance:** This shows what the telescope **actually observes**. \"\n        \"The raw data is a time series of these 32×32 pixel images, flattened into 1024-element vectors. \"\n        \"The bright horizontal band is **starlight dispersed by the spectrograph**. \"\n        \"**The planet's atmospheric signal is hidden in the subtle changes of this trace over time.**\"\n    ))\n\nexcept FileNotFoundError:\n    print(f\"⚠️ Could not find FGS1 data for planet {SAMPLE_PLANET_ID}\")\n    print(\"This planet may not have FGS1 observations available.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-20T17:00:59.378943Z","iopub.execute_input":"2025-07-20T17:00:59.379624Z","iopub.status.idle":"2025-07-20T17:01:01.731881Z","shell.execute_reply.started":"2025-07-20T17:00:59.37943Z","shell.execute_reply":"2025-07-20T17:01:01.730366Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 5.3 Time Series Analysis: Signal vs. Noise ---\ndisplay_header(\"📈 Signal vs. Noise Analysis\")\n\ndef extract_time_data(axis_info_df, signal_length, instrument='FGS1'):\n    \"\"\"\n    Time data extraction handling various column name formats\n    \"\"\"\n    print(f\"🔍 Extracting time data for {instrument}...\")\n    print(f\"📊 Available columns: {axis_info_df.columns.tolist()}\")\n    \n    # Looking for exact instrument-specific column\n    target_patterns = [\n        f'{instrument}-axis0-h',\n        f'{instrument}-axis0',\n        f'time_{instrument}',\n        f'{instrument}_time'\n    ]\n    \n    for pattern in target_patterns:\n        if pattern in axis_info_df.columns:\n            print(f\"✅ Found time column: {pattern}\")\n            time_data = axis_info_df[pattern].values\n            return time_data[:signal_length] if len(time_data) >= signal_length else time_data\n    \n    print(\"⚠️ Generating synthetic time array (dummy data)\")\n    return np.linspace(0, signal_length-1, signal_length)\n\n\nif fgs1_signal is not None:\n    # Extract time data\n    time_data = extract_time_data(axis_info_df, len(fgs1_signal), 'FGS1')\n    \n    # Converting to seconds if in hours\n    if time_data.max() < 1.0:  # Likely in hours\n        time_data_seconds = time_data * 3600\n        time_unit = \"seconds\"\n    else:\n        time_data_seconds = time_data\n        time_unit = \"time units\"\n    \n    # Select pixel for analysis (center of detector)\n    pixel_idx = 512  # Center pixel\n    pixel_flux = fgs1_signal.iloc[:, pixel_idx].values\n    \n    # Quality check\n    valid_indices = ~np.isnan(time_data_seconds) & ~np.isnan(pixel_flux)\n    n_valid = valid_indices.sum()\n    \n    print(f\"📊 Time Series Statistics:\")\n    print(f\"   • Total time points: {len(time_data_seconds):,}\")\n    print(f\"   • Valid data points: {n_valid:,}\")\n    print(f\"   • Observation duration: {time_data_seconds.max():.1f} {time_unit}\")\n    print(f\"   • Pixel analyzed: {pixel_idx} (center)\")\n    \n    if n_valid > 10:  # trying only 10 data for analysis\n        # Fit polynomial trend (instrument systematics)\n        poly_degree = 3  # Slightly higher order for better fit\n        poly_coeffs = np.polyfit(time_data_seconds[valid_indices], pixel_flux[valid_indices], poly_degree)\n        poly_fit = np.poly1d(poly_coeffs)\n        trend_line = poly_fit(time_data_seconds[valid_indices])\n        \n        # Calculate detrended signal\n        detrended_signal = pixel_flux - poly_fit(time_data_seconds)\n        \n        # Transit detection (approximate)\n        min_flux_idx = np.nanargmin(pixel_flux)\n        transit_buffer = min(20000, len(pixel_flux) // 10)\n        transit_start = max(0, min_flux_idx - transit_buffer)\n        transit_end = min(len(pixel_flux) - 1, min_flux_idx + transit_buffer)\n        \n        # Create comprehensive visualization\n        fig, axes = plt.subplots(2, 2, figsize=(18, 12))\n        \n        # Plot 1: Raw signal with trend\n        ax1 = axes[0, 0]\n        ax1.plot(time_data_seconds, pixel_flux, 'b-', alpha=0.7, linewidth=1, \n                label='Raw Signal')\n        ax1.plot(time_data_seconds[valid_indices], trend_line, 'g--', \n                linewidth=2, label='Systematic Trend')\n        ax1.axvspan(time_data_seconds[transit_start], time_data_seconds[transit_end], \n                   color='orange', alpha=0.2, label='Transit Region')\n        ax1.set_xlabel(f'Time ({time_unit})')\n        ax1.set_ylabel('Flux (ADC Units)')\n        ax1.set_title('Raw Signal + Instrumental Trend')\n        ax1.legend()\n        ax1.grid(True, alpha=0.3)\n        \n        # Plot 2: Detrended signal\n        ax2 = axes[0, 1]\n        ax2.plot(time_data_seconds, detrended_signal, 'r-', alpha=0.7, linewidth=1,\n                label='Detrended Signal')\n        ax2.axvspan(time_data_seconds[transit_start], time_data_seconds[transit_end], \n                   color='orange', alpha=0.2, label='Transit Region')\n        ax2.set_xlabel(f'Time ({time_unit})')\n        ax2.set_ylabel('Relative Flux')\n        ax2.set_title('Detrended Signal (Transit Visible)')\n        ax2.legend()\n        ax2.grid(True, alpha=0.3)\n        \n        # Plot 3: Transit zoom-in\n        ax3 = axes[1, 0]\n        zoom_margin = transit_buffer // 2\n        zoom_start = max(0, min_flux_idx - zoom_margin)\n        zoom_end = min(len(pixel_flux) - 1, min_flux_idx + zoom_margin)\n        \n        ax3.plot(time_data_seconds[zoom_start:zoom_end], \n                detrended_signal[zoom_start:zoom_end], \n                'r-', linewidth=2, label='Transit Signal')\n        ax3.axhline(y=0, color='k', linestyle='--', alpha=0.5)\n        ax3.set_xlabel(f'Time ({time_unit})')\n        ax3.set_ylabel('Relative Flux')\n        ax3.set_title('Transit Event (Zoomed)')\n        ax3.legend()\n        ax3.grid(True, alpha=0.3)\n        \n        # Plot 4: Noise analysis\n        ax4 = axes[1, 1]\n        residuals = pixel_flux[valid_indices] - trend_line\n        ax4.hist(residuals, bins=50, alpha=0.7, color='purple', edgecolor='black')\n        ax4.axvline(x=0, color='k', linestyle='--', alpha=0.5)\n        ax4.set_xlabel('Residuals (ADC Units)')\n        ax4.set_ylabel('Frequency')\n        ax4.set_title('Noise Distribution')\n        ax4.grid(True, alpha=0.3)\n        \n        plt.tight_layout()\n        plt.show()\n        \n        # Calculate performance metrics\n        signal_range = np.ptp(pixel_flux)\n        noise_std = np.std(residuals)\n        transit_depth = np.min(detrended_signal[transit_start:transit_end])\n        \n        print(f\"📊 Signal Quality Metrics:\")\n        print(f\"   • Signal range: {signal_range:.1f} ADC units\")\n        print(f\"   • Noise std: {noise_std:.2f} ADC units\")\n        print(f\"   • Approximate SNR: {signal_range/noise_std:.1f}\")\n        print(f\"   • Transit depth: {abs(transit_depth):.4f} (relative)\")\n        print(f\"   • Transit duration: {time_data_seconds[transit_end] - time_data_seconds[transit_start]:.1f} {time_unit}\")\n        \n        display(Markdown(\n            \"**💡 Significance:** This analysis reveals the **core challenge**:\\n\\n\"\n            \"- **Blue line (top-left)**: Raw telescope data showing instrumental drift\\n\"\n            \"- **Red line (top-right)**: Detrended signal revealing the transit\\n\"\n            \"- **Bottom-left**: Zoomed transit showing the atmospheric signature\\n\"\n            \"- **Bottom-right**: Noise characteristics for filter design\\n\\n\"\n            \"**The task is to separate the tiny planetary signal from complex systematic noise.**\"\n        ))\n    else:\n        print(\"⚠️ Insufficient valid data points for time series analysis\")\n            \nelse:\n    print(\"⚠️ No FGS1 signal data available for time series analysis\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-20T17:01:01.73325Z","iopub.execute_input":"2025-07-20T17:01:01.733684Z","iopub.status.idle":"2025-07-20T17:01:06.870813Z","shell.execute_reply.started":"2025-07-20T17:01:01.733648Z","shell.execute_reply":"2025-07-20T17:01:06.869816Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 🎨 SECTION 6: VISUALIZATIONS","metadata":{}},{"cell_type":"code","source":"# ====================================================================\n# 🎨 SECTION 6: VISUALIZATIONS\n# ====================================================================\n\ndisplay_header(\"🎨 Visualizations\")\n\ndef plot_feature_distributions():\n    \"\"\"Plot distributions of stellar/planetary parameters\"\"\"\n    feature_cols = ['Rs', 'Ms', 'Ts', 'Mp', 'P', 'sma', 'i']\n    palette = sns.color_palette(\"viridis\", len(feature_cols))\n    \n    fig, axes = plt.subplots(nrows=2, ncols=4, figsize=(20, 10))\n    axes = axes.flatten()\n    \n    for i, col in enumerate(feature_cols):\n        ax = axes[i]\n        sns.histplot(\n            data=merged_train_df,\n            x=col,\n            ax=ax,\n            kde=True,\n            color=palette[i]\n        )\n        ax.set_title(f'Distribution of {col}', fontsize=14)\n        ax.set_xlabel('')\n    \n    fig.suptitle('Stellar and Planetary Parameter Distributions', fontsize=20, y=1.02)\n    plt.tight_layout()\n    plt.show()\n\ndef plot_sample_spectrum():\n    \"\"\"Plot ground truth spectrum for sample planet\"\"\"\n    # Sample planet data\n    planet_sample = merged_train_df[merged_train_df['planet_id'] == int(SAMPLE_PLANET_ID)]\n    \n    if len(planet_sample) == 0:\n        planet_sample = merged_train_df.iloc[[0]]\n        sample_id = planet_sample['planet_id'].values[0]\n    else:\n        sample_id = int(SAMPLE_PLANET_ID)\n    \n    # Spectrum data\n    wl_cols = [col for col in train_df.columns if col.startswith('wl_')]\n    spectrum_data = planet_sample.melt(\n        id_vars=['planet_id'], \n        value_vars=wl_cols,\n        var_name='wl_id', \n        value_name='spectrum_value'\n    )\n    \n    # Merge with wavelengths\n    spectrum_plot = pd.merge(spectrum_data, wavelengths_long_df, on='wl_id')\n    \n    # Plot\n    plt.figure(figsize=(16, 8))\n    \n    instrument_colors = {\"FGS1\": \"#0077b6\", \"AIRS-CH0\": \"#d62728\"}\n    \n    sns.lineplot(\n        data=spectrum_plot,\n        x='wavelength_um',\n        y='spectrum_value',\n        hue='instrument',\n        style='instrument',\n        markers=True,\n        dashes=False,\n        palette=instrument_colors,\n        linewidth=2,\n        markersize=8\n    )\n    \n    plt.title(f'Ground Truth Spectrum - Planet ID: {sample_id}', fontsize=18)\n    plt.xlabel('Wavelength (μm)', fontsize=14)\n    plt.ylabel('Transit Depth', fontsize=14)\n    plt.legend(title='Instrument', fontsize=12)\n    plt.grid(True, alpha=0.3)\n    plt.show()\n\n# Visualizations\nplot_feature_distributions()\nplot_sample_spectrum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-20T17:01:06.874115Z","iopub.execute_input":"2025-07-20T17:01:06.874454Z","iopub.status.idle":"2025-07-20T17:01:09.484054Z","shell.execute_reply.started":"2025-07-20T17:01:06.874431Z","shell.execute_reply":"2025-07-20T17:01:09.482743Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 🔬 SECTION 7: SIGNAL PROCESSING ANALYSIS","metadata":{}},{"cell_type":"code","source":"# ====================================================================\n# 🔬 SECTION 7: SIGNAL PROCESSING ANALYSIS\n# ====================================================================\ndisplay_header(\"🔬 Signal Processing Analysis\")\n\ndef differential_reading(light_curve):\n    \"\"\"differential reading with comprehensive bounds checking\"\"\"\n    \n    # Ensureing we have enough data for differential reading\n    if len(light_curve) < 2:\n        return np.array([])\n    \n    # Calculate valid start indices (even positions)\n    max_start_idx = len(light_curve) - 2  # some room for end_idx\n    start_indices = np.arange(0, max_start_idx + 1, 2)\n    \n    # Calculate corresponding end indices\n    end_indices = start_indices + 1\n    \n    # ensuring all indices are valid\n    valid_mask = (end_indices < len(light_curve)) & (start_indices >= 0)\n    start_indices = start_indices[valid_mask]\n    end_indices = end_indices[valid_mask]\n    \n    # Perform differential reading\n    if len(start_indices) > 0:\n        diff = light_curve.iloc[end_indices].values - light_curve.iloc[start_indices].values\n        return diff\n    else:\n        return np.array([])\n\ndef process_raw_signal(planet_id):\n    \"\"\" Signal processing pipeline for a single planet\"\"\"\n    \n    print(f\"🔄 Processing Planet {planet_id}...\")\n    \n    # Load raw signal data\n    fgs1_path = BASE_PATH / f\"train/{planet_id}/FGS1_signal_0.parquet\"\n    airs_path = BASE_PATH / f\"train/{planet_id}/AIRS-CH0_signal_0.parquet\"\n    \n    try:\n        fgs1_raw = pd.read_parquet(fgs1_path)\n        airs_raw = pd.read_parquet(airs_path)\n        \n        print(f\"   ✅ Raw data loaded: FGS1 {fgs1_raw.shape}, AIRS-CH0 {airs_raw.shape}\")\n        \n        # ADC calibration\n        fgs1_calibrated = adc_calibration(fgs1_raw, gain, offset)\n        airs_calibrated = adc_calibration(airs_raw, gain, offset)\n        \n        # Create light curves (sum all pixels)\n        fgs1_light_curve = fgs1_calibrated.sum(axis=1)\n        airs_light_curve = airs_calibrated.sum(axis=1)\n        \n        print(f\"   ✅ Light curves created\")\n        \n        fgs1_diff = differential_reading(fgs1_light_curve)\n        airs_diff = differential_reading(airs_light_curve)\n        \n        print(f\"   ✅ Differential reading applied\")\n        print(f\"       • FGS1 differential points: {len(fgs1_diff)}\")\n        print(f\"       • AIRS-CH0 differential points: {len(airs_diff)}\")\n        \n        # Proceed with signal cleaning for AIRS-CH0 (better signal)\n        if len(airs_diff) > 0:\n            airs_cleaned, outliers = sigma_clip(airs_diff)\n            \n            # Transit detection\n            transit_start, transit_end, transit_depth = detect_transit(airs_cleaned)\n            \n            print(f\"   ✅ Transit detected: frames {transit_start}-{transit_end}, depth {abs(transit_depth):.6f}\")\n            \n            # Detrending\n            out_of_transit_mask = np.ones_like(airs_cleaned, dtype=bool)\n            out_of_transit_mask[transit_start:transit_end] = False\n            out_of_transit_mask[np.isnan(airs_cleaned)] = False\n            \n            x_axis = np.arange(len(airs_cleaned))\n            poly_coeffs = np.polyfit(x_axis[out_of_transit_mask], airs_cleaned[out_of_transit_mask], 2)\n            baseline_model = np.polyval(poly_coeffs, x_axis)\n            airs_normalized = airs_cleaned / baseline_model\n            \n            print(f\"   ✅ Signal processing complete\")\n            \n            return {\n                'fgs1_raw': fgs1_raw,\n                'airs_raw': airs_raw,\n                'fgs1_diff': fgs1_diff,\n                'airs_diff': airs_diff,\n                'airs_cleaned': airs_cleaned,\n                'airs_normalized': airs_normalized,\n                'baseline_model': baseline_model,\n                'transit_range': (transit_start, transit_end),\n                'outliers': outliers\n            }\n        else:\n            print(\"   ❌ No differential data points available\")\n            return None\n            \n    except Exception as e:\n        print(f\"   ❌ Error processing planet {planet_id}: {e}\")\n        return None\n\n\ndef visualize_detector_images(fgs1_raw, airs_raw, planet_id, frame_idx=5000):\n    \"\"\"Visualize detector images\"\"\"\n    # Get sample frames\n    fgs1_frame = adc_calibration(fgs1_raw.iloc[frame_idx], gain, offset)\n    airs_frame = adc_calibration(airs_raw.iloc[frame_idx], gain, offset)\n    \n    # Reshape to 2D\n    fgs1_2d = fgs1_frame.values.reshape(32, 32)\n    airs_2d = airs_frame.values.reshape(32, 356)\n    \n    # Create visualization\n    fig, axes = plt.subplots(2, 1, figsize=(18, 12))\n    \n    # FGS1 detector\n    sns.heatmap(fgs1_2d, ax=axes[0], cmap='viridis', cbar_kws={'label': 'Flux'})\n    axes[0].set_title(f'FGS1 Detector Image (32×32) - Planet {planet_id}, Frame {frame_idx}', fontsize=16)\n    axes[0].set_xlabel('Pixel Column')\n    axes[0].set_ylabel('Pixel Row')\n    \n    # AIRS-CH0 detector\n    sns.heatmap(airs_2d, ax=axes[1], cmap='viridis', cbar_kws={'label': 'Flux'})\n    axes[1].set_title(f'AIRS-CH0 Detector Image (32×356) - Planet {planet_id}, Frame {frame_idx}', fontsize=16)\n    axes[1].set_xlabel('Wavelength Axis (Pixel Column)')\n    axes[1].set_ylabel('Spatial Axis (Pixel Row)')\n    \n    plt.tight_layout()\n    plt.show()\n\ndef visualize_signal_processing(results, planet_id):\n    \"\"\"Visualize complete signal processing pipeline\"\"\"\n    \n    if results is None:\n        print(\"❌ No results to visualize\")\n        return\n    \n    # Extract data\n    fgs1_diff = results['fgs1_diff']\n    airs_diff = results['airs_diff']\n    airs_cleaned = results['airs_cleaned']\n    airs_normalized = results['airs_normalized']\n    baseline_model = results['baseline_model']\n    transit_range = results['transit_range']\n    outliers = results['outliers']\n    \n    # Create comprehensive visualization\n    fig, axes = plt.subplots(2, 2, figsize=(20, 12))\n    \n    # Plot 1: Raw differential signals\n    # Create separate x-axes for each instrument\n    fgs1_x_axis = np.arange(len(fgs1_diff))\n    airs_x_axis = np.arange(len(airs_diff))\n    \n    # Plot FGS1 and AIRS-CH0 separately with their own x-axes\n    axes[0,0].plot(fgs1_x_axis, fgs1_diff, color='#0077b6', linewidth=0.8, alpha=0.7, label='FGS1')\n    axes[0,0].plot(airs_x_axis, airs_diff, color='#d62728', linewidth=0.8, alpha=0.7, label='AIRS-CH0')\n    \n    # Only showing transit region on AIRS-CH0\n    axes[0,0].axvspan(transit_range[0], transit_range[1], color='orange', alpha=0.3, label='Transit (AIRS-CH0)')\n    axes[0,0].set_title('Raw Differential Signals', fontsize=14)\n    axes[0,0].set_xlabel('Frame Number')\n    axes[0,0].set_ylabel('Differential Flux')\n    axes[0,0].legend()\n    axes[0,0].grid(True, alpha=0.3)\n    \n    # Plot 2: Sigma clipping and baseline - Use AIRS-CH0 x-axis\n    axes[0,1].plot(airs_x_axis, airs_diff, color='lightgray', linewidth=0.5, alpha=0.7, label='Raw')\n    axes[0,1].plot(airs_x_axis, baseline_model, color='orange', linewidth=2, label='Baseline')\n    axes[0,1].scatter(airs_x_axis[outliers], airs_diff[outliers], color='red', s=20, label='Outliers', zorder=5)\n    axes[0,1].set_title('Sigma Clipping & Baseline Fitting', fontsize=14)\n    axes[0,1].set_xlabel('Frame Number')\n    axes[0,1].set_ylabel('Flux')\n    axes[0,1].legend()\n    axes[0,1].grid(True, alpha=0.3)\n    \n    # Plot 3: Final normalized light curve - Use AIRS-CH0 x-axis\n    axes[1,0].plot(airs_x_axis, airs_normalized, color='dodgerblue', linewidth=1.5, label='Normalized')\n    axes[1,0].axvspan(transit_range[0], transit_range[1], color='orange', alpha=0.3, label='Transit')\n    axes[1,0].set_title('Final Normalized Light Curve', fontsize=14)\n    axes[1,0].set_xlabel('Frame Number')\n    axes[1,0].set_ylabel('Normalized Flux')\n    axes[1,0].set_ylim(0.98, 1.01)\n    axes[1,0].legend()\n    axes[1,0].grid(True, alpha=0.3)\n    \n    # Plot 4: Transit zoom - Use AIRS-CH0 x-axis\n    buffer = (transit_range[1] - transit_range[0]) // 3\n    zoom_start = max(0, transit_range[0] - buffer)\n    zoom_end = min(len(airs_normalized) - 1, transit_range[1] + buffer)\n    \n    axes[1,1].plot(airs_x_axis[zoom_start:zoom_end], airs_normalized[zoom_start:zoom_end], \n                   color='red', linewidth=2, label='Transit Signal')\n    axes[1,1].axhline(y=1.0, color='black', linestyle='--', alpha=0.5)\n    axes[1,1].set_title('Transit Event (Zoomed)', fontsize=14)\n    axes[1,1].set_xlabel('Frame Number')\n    axes[1,1].set_ylabel('Normalized Flux')\n    axes[1,1].legend()\n    axes[1,1].grid(True, alpha=0.3)\n    \n    fig.suptitle(f'Complete Signal Processing Pipeline - Planet {planet_id}', fontsize=18)\n    plt.tight_layout()\n    plt.show()\n\n# Run complete processing on sample planet\ntry:\n    # Find a planet that exists in the data\n    planet_folders = [p.name for p in (BASE_PATH / 'train').iterdir() if p.is_dir()]\n    \n    if SAMPLE_PLANET_ID in planet_folders:\n        demo_planet = SAMPLE_PLANET_ID\n    else:\n        demo_planet = planet_folders[0]  # Use first available planet\n    \n    print(f\"Running demo on Planet {demo_planet}...\")\n    \n    # Process the planet\n    results = process_raw_signal(demo_planet)\n    \n    if results is not None:\n        # Visualize detector images\n        visualize_detector_images(results['fgs1_raw'], results['airs_raw'], demo_planet)\n        \n        # Visualize signal processing\n        visualize_signal_processing(results, demo_planet)\n        \n        # Calculate quality metrics\n        transit_signal = results['airs_normalized'][results['transit_range'][0]:results['transit_range'][1]]\n        transit_depth = 1.0 - np.nanmean(transit_signal)\n        noise_level = np.nanstd(results['airs_normalized'])\n        snr = transit_depth / noise_level if noise_level > 0 else 0\n        \n        print(f\"📊 Quality Metrics for Planet {demo_planet}:\")\n        print(f\"   • Transit depth: {transit_depth:.6f}\")\n        print(f\"   • Noise level: {noise_level:.6f}\")\n        print(f\"   • SNR: {snr:.2f}\")\n        print(f\"   • Data points: {len(results['airs_normalized'])}\")\n        print(f\"   • Outliers removed: {results['outliers'].sum()}\")\n        \nexcept Exception as e:\n    print(f\"❌ Error in signal processing demo: {e}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-20T17:01:09.485249Z","iopub.execute_input":"2025-07-20T17:01:09.48556Z","iopub.status.idle":"2025-07-20T17:01:17.089684Z","shell.execute_reply.started":"2025-07-20T17:01:09.485539Z","shell.execute_reply":"2025-07-20T17:01:17.088079Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 🔍 SECTION 8: SUMMARY","metadata":{}},{"cell_type":"code","source":"# ====================================================================\n# 🔍 SECTION 8: Summary\n# ====================================================================\n\ndisplay_header(\"🎯 Summary\")\n\n# Dataset scale analysis\ntotal_planets = len(train_df)\ntotal_wavelengths = len([col for col in train_df.columns if col.startswith('wl_')])\ntest_planets = len(sample_submission_df)\n\nprint(\"📊 DATASET CHARACTERISTICS:\")\nprint(f\"   • Training planets: {len(train_df):,}\")\nprint(f\"   • Test planets: {test_planets:,}\")\nprint(f\"   • Spectral points: {len(wavelengths)}\")\nprint(f\"   • Wavelength range: {wavelengths.min():.3f} - {wavelengths.max():.3f} μm\")\nprint(f\"   • Time axis shape: {axis_info_df.shape}\")\nprint(f\"   • Estimated dataset size: ~263 GB\")\n\nprint(f\"\\n🛰️ INSTRUMENT ANALYSIS:\")\nprint(f\"   • FGS1 (photometer): {fgs1_count} points at visible wavelengths\")\nprint(f\"   • AIRS-CH0 (spectrometer): {airs_count} points at infrared wavelengths\")\nprint(f\"   • Multi-instrument processing required\")\n\n# Instrument coverage analysis\nwavelength_ranges = {\n    'FGS1': (0.6, 0.8),\n    'AIRS-CH0': (1.95, 3.9)\n}\n\nfor instrument, (min_wl, max_wl) in wavelength_ranges.items():\n    mask = (wavelengths >= min_wl) & (wavelengths <= max_wl)\n    coverage = mask.sum()\n    print(f\"   • {instrument} coverage: {coverage} points ({min_wl}-{max_wl} μm)\")\n\nprint(\"=\" * 70)\n\nprint(\"📋 FINDINGS:\")\nprint(f\"   ✅ Dataset scale: {total_planets:,} planets, {total_wavelengths} wavelengths\")\nprint(f\"   ✅ Time structure: {axis_info_df.shape} with instrument-specific axes\")\nprint(f\"   ✅ Dual instruments: FGS1 + AIRS-CH0 coverage confirmed\")\nprint(f\"   ✅ ADC conversion: Parameters available for physical units\")\nprint(f\"   ✅ Rich metadata: Stellar/planetary features ready for use\")\nprint(f\"   ✅ Submission format: 283 + 283 dual predictions required\")\n\nprint(f\"Reference:\")\nprint(f\"   https://www.kaggle.com/code/lordpatil/perfect-eda-doesn-t-exist\")\n\nprint(\"=\" * 70)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-20T17:01:17.091247Z","iopub.execute_input":"2025-07-20T17:01:17.091641Z","iopub.status.idle":"2025-07-20T17:01:17.105865Z","shell.execute_reply.started":"2025-07-20T17:01:17.091608Z","shell.execute_reply":"2025-07-20T17:01:17.104659Z"}},"outputs":[],"execution_count":null}]}