{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":56537,"databundleVersionId":8877088,"sourceType":"competition"}],"dockerImageVersionId":30715,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-27T19:26:52.2763Z","iopub.execute_input":"2024-06-27T19:26:52.276688Z","iopub.status.idle":"2024-06-27T19:26:52.284608Z","shell.execute_reply.started":"2024-06-27T19:26:52.276657Z","shell.execute_reply":"2024-06-27T19:26:52.283266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nimport numpy as np\nimport seaborn as sns\nimport sklearn as sk\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nimport torch\nfrom torch.utils.data import DataLoader, TensorDataset\nfrom statistics import mean\nfrom xgboost import XGBRegressor\nfrom sklearn.metrics import r2_score\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import mean_squared_error","metadata":{"execution":{"iopub.status.busy":"2024-06-27T19:31:58.859301Z","iopub.execute_input":"2024-06-27T19:31:58.859692Z","iopub.status.idle":"2024-06-27T19:31:58.866661Z","shell.execute_reply.started":"2024-06-27T19:31:58.859659Z","shell.execute_reply":"2024-06-27T19:31:58.865295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"leap_raw_df = pl.scan_csv(f'/kaggle/input/leap-atmospheric-physics-ai-climsim/train.csv')\nfetch_size =1000\ninput_columns = leap_raw_df.columns[1:557]\noutput_columns = leap_raw_df.columns[557:]\nX = leap_raw_df.select(pl.col(input_columns)).fetch(fetch_size)\nY = leap_raw_df.select(pl.col(output_columns)).fetch(fetch_size)\nbig = leap_raw_df.select(pl.col(\"*\")).fetch(fetch_size)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T19:26:52.30027Z","iopub.execute_input":"2024-06-27T19:26:52.300649Z","iopub.status.idle":"2024-06-27T19:26:52.565386Z","shell.execute_reply.started":"2024-06-27T19:26:52.300611Z","shell.execute_reply":"2024-06-27T19:26:52.56406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def adicionar_agua_v_erro_info(big: pl.DataFrame, margen = 0.00005) ->pl.DataFrame:\n    gravidade = 9.80665\n    ptend_vapor = big.select(pl.col(\"^ptend_q0001.*$\")).to_numpy()\n    ptend_liquido = big.select(pl.col(\"^ptend_q0002.*$\")).to_numpy()\n    ptend_gelo = big.select(pl.col(\"^ptend_q0003.*$\")).to_numpy()\n    \n    state_vapor = big.select(pl.col(\"^state_q0001.*$\")).to_numpy()\n    state_liquido = big.select(pl.col(\"^state_q0002.*$\")).to_numpy()\n    state_gelo = big.select(pl.col(\"^state_q0003.*$\")).to_numpy()\n    \n    precipitacao_agua = big.select(pl.col(\"^cam_out_PRECC.*$\")).to_numpy().flatten()\n    precipitacao_neve = big.select(pl.col(\"^cam_out_PRECSC.*$\")).to_numpy().flatten()\n    \n    \n    pressao = big.select(pl.col(\"^state_ps.*$\")).to_numpy().flatten()\n    volume_inicial = np.sum(state_vapor + state_liquido + state_gelo,axis=1)*(pressao/gravidade)\n    volume_predito = np.sum(ptend_vapor + ptend_liquido + ptend_gelo,axis=1)*(pressao/gravidade) + precipitacao_agua + precipitacao_neve\n    \n    \n    erro_volumetrico = abs((volume_predito - volume_inicial)/volume_inicial +1)\n    flag_erro_volumetrico = (erro_volumetrico >margen).astype(np.int8)\n    big = big.with_columns([\n        pl.Series(\"erro_volumetrico\",erro_volumetrico),\n        pl.Series(\"flag_erro_volumetrico\",flag_erro_volumetrico)\n    ])\n    return big","metadata":{"execution":{"iopub.status.busy":"2024-06-27T19:26:52.567828Z","iopub.execute_input":"2024-06-27T19:26:52.568444Z","iopub.status.idle":"2024-06-27T19:26:52.581156Z","shell.execute_reply.started":"2024-06-27T19:26:52.568404Z","shell.execute_reply":"2024-06-27T19:26:52.579917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\ndef custom_log_transform(column):\n    \"\"\"Applies a log1p transformation to a given column with an adjustment for negative values.\"\"\"\n    min_value = column.min()\n    transformed_column = np.log1p(column - min_value + 1)  # log1p is log(1 + x) which is stable for small x\n    return transformed_column\n\ndef log_transformation(data):\n    \"\"\"Applies custom log transformation to each column in the data.\"\"\"\n    data_log_transformed = np.array(data, dtype=\"float64\")\n    for i in range(data.shape[1]):\n        data_log_transformed[:, i] = custom_log_transform(data[:, i])\n    return data_log_transformed\n\ndef preprocess_data(X, Y, transform=True, log=False):\n    \"\"\"Preprocesses the data by scaling and optionally applying log transformation.\"\"\"\n    X_np = X.to_numpy()\n    Y_np = Y.to_numpy()\n\n    if transform:\n        # Standardize features\n        X_scaler = StandardScaler().fit(X_np)\n        Y_scaler = StandardScaler().fit(Y_np)\n\n        X_transformed = X_scaler.transform(X_np)\n        Y_transformed = Y_scaler.transform(Y_np)\n\n        if log:\n            X_transformed = log_transformation(X_transformed)\n            Y_transformed = log_transformation(Y_transformed)\n        \n        # Clean up\n        del X, Y, X_np, Y_np\n\n    else:\n        # Split data into training and validation sets without transformation\n        #X_train, X_val, Y_train, Y_val = train_test_split(X_np, Y_np, test_size=0.2, random_state=42)\n        \n        # Clean up\n        del X, Y, X_np, Y_np\n\n    return X_transformed, Y_transformed\n\n# Example usage\n# Assuming X and Y are defined and are Polars DataFrames\nX_process, Y_process = preprocess_data(X, Y, transform=True, log=False)\nX_train, X_val, Y_train, Y_val = train_test_split(X_process, Y_process, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T19:26:52.582663Z","iopub.execute_input":"2024-06-27T19:26:52.583028Z","iopub.status.idle":"2024-06-27T19:26:52.628512Z","shell.execute_reply.started":"2024-06-27T19:26:52.582971Z","shell.execute_reply":"2024-06-27T19:26:52.627377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"big=adicionar_agua_v_erro_info(big)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T19:26:52.630637Z","iopub.execute_input":"2024-06-27T19:26:52.630991Z","iopub.status.idle":"2024-06-27T19:26:52.64359Z","shell.execute_reply.started":"2024-06-27T19:26:52.630951Z","shell.execute_reply":"2024-06-27T19:26:52.642539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.boxplot(big[\"erro_volumetrico\"])","metadata":{"execution":{"iopub.status.busy":"2024-06-27T19:26:52.645167Z","iopub.execute_input":"2024-06-27T19:26:52.64559Z","iopub.status.idle":"2024-06-27T19:26:52.859171Z","shell.execute_reply.started":"2024-06-27T19:26:52.645551Z","shell.execute_reply":"2024-06-27T19:26:52.858004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sum(big[\"flag_erro_volumetrico\"]>0)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T19:26:52.860441Z","iopub.execute_input":"2024-06-27T19:26:52.860823Z","iopub.status.idle":"2024-06-27T19:26:52.86781Z","shell.execute_reply.started":"2024-06-27T19:26:52.860753Z","shell.execute_reply":"2024-06-27T19:26:52.866796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.boxplot(big[\"erro_volumetrico\"])","metadata":{"execution":{"iopub.status.busy":"2024-06-27T19:26:52.869013Z","iopub.execute_input":"2024-06-27T19:26:52.869332Z","iopub.status.idle":"2024-06-27T19:26:53.041359Z","shell.execute_reply.started":"2024-06-27T19:26:52.869305Z","shell.execute_reply":"2024-06-27T19:26:53.040245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Initialize model\nmodel = XGBRegressor(\n    n_estimators= 1150,\n    max_depth=56,\n    learning_rate=0.003,\n    colsample_bytree=0.9,\n    subsample=0.8,\n    min_child_weight=1,\n    random_state=42,\n    objective='reg:squarederror',\n    n_jobs=-1,  # Use all available cores\n    tree_method='hist',\n    device ='cuda',\n)\n\nr2_vector = []\nmse_vector = []\n\ny_pred_df = pd.DataFrame()\nout_i =36\nout_f=361\nfor i in range(out_i, out_f):\n    \n    model.fit(X_train, Y_train[:, i]) \n    y_pred = model.predict(X_val)\n    y_pred_df[f'y_pred_{i}'] = y_pred\n    \n    # Calculate r\n    r2 = r2_score(Y_val[:, i], y_pred)\n    mse = mean_squared_error(Y_val[:, i], y_pred)\n    print(f\"{r2},{mse}\")\n    # Append scores to the lists\n    r2_vector.append(r2)\n    mse_vector.append(mse)\n\n# Optionally convert lists to DataFrame for r2 and mse scores\nr2_df = pd.DataFrame({'r2_score': r2_vector})\nmse_df = pd.DataFrame({'mse_score': mse_vector})\n\n# Now y_pred_df contains all the predictions for each target variable\n","metadata":{"execution":{"iopub.status.busy":"2024-06-27T19:27:36.042382Z","iopub.execute_input":"2024-06-27T19:27:36.042774Z","iopub.status.idle":"2024-06-27T19:30:24.98786Z","shell.execute_reply.started":"2024-06-27T19:27:36.042742Z","shell.execute_reply":"2024-06-27T19:30:24.9862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = output_columns[out_i:out_f]\nplt.figure(figsize=(200, 6))\nplt.plot(columns, mse_vector, label='MSE', color='blue', marker='o')\nplt.plot(columns, r2_vector, label='R²', color='red', marker='x')\n\nplt.xlabel('Column Number')\nplt.ylabel('Score')\nplt.title('MSE and R² for each column')\nplt.legend()\nplt.grid(True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-27T19:30:24.988862Z","iopub.status.idle":"2024-06-27T19:30:24.989403Z","shell.execute_reply.started":"2024-06-27T19:30:24.989137Z","shell.execute_reply":"2024-06-27T19:30:24.989161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}