{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":56537,"databundleVersionId":8015876,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\n\n\ndef check_unique(data: pl.Series) -> tuple[bool, float]:\n    unique_list = data.unique().to_list()\n    unique_flag = True if len(unique_list) == 1 else False\n    return unique_flag, unique_list[0]","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-07T06:14:04.727602Z","iopub.execute_input":"2024-05-07T06:14:04.728182Z","iopub.status.idle":"2024-05-07T06:14:04.838707Z","shell.execute_reply.started":"2024-05-07T06:14:04.728142Z","shell.execute_reply":"2024-05-07T06:14:04.837471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# invalid targets <==> weight == 0\ndf_weight = pd.read_csv(\"/kaggle/input/leap-atmospheric-physics-ai-climsim/sample_submission.csv\", nrows=1)\ninvalid_targets: list[str] = []\nfor col in df_weight.columns.to_list():\n    if col == \"sample_id\":\n        continue\n    if df_weight[col].values[0] == 0.0:\n        invalid_targets.append(col)\n        \nprint(invalid_targets[:5], len(invalid_targets))","metadata":{"execution":{"iopub.status.busy":"2024-05-07T06:06:48.006027Z","iopub.execute_input":"2024-05-07T06:06:48.006494Z","iopub.status.idle":"2024-05-07T06:06:48.082979Z","shell.execute_reply.started":"2024-05-07T06:06:48.006463Z","shell.execute_reply":"2024-05-07T06:06:48.08175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\n# save invalid targets as json file\nwith open(\"invalid_targets.json\", \"w\") as f:\n    json.dump(invalid_targets, f)","metadata":{"execution":{"iopub.status.busy":"2024-05-07T06:08:47.565668Z","iopub.execute_input":"2024-05-07T06:08:47.566166Z","iopub.status.idle":"2024-05-07T06:08:47.573414Z","shell.execute_reply.started":"2024-05-07T06:08:47.566127Z","shell.execute_reply":"2024-05-07T06:08:47.571894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\n# test unique features\ndf_test = pl.read_csv(\"/kaggle/input/leap-atmospheric-physics-ai-climsim/test.csv\")\ntest_unique_cols = {}\nfor col in tqdm(df_test.columns):\n    if col == \"sample_id\":\n        continue\n    unique_flag, unique_value = check_unique(df_test[col])\n    if unique_flag:\n        test_unique_cols[col] = unique_value","metadata":{"execution":{"iopub.status.busy":"2024-05-07T06:18:09.879412Z","iopub.execute_input":"2024-05-07T06:18:09.879944Z","iopub.status.idle":"2024-05-07T06:19:11.486891Z","shell.execute_reply.started":"2024-05-07T06:18:09.879908Z","shell.execute_reply":"2024-05-07T06:19:11.485671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print([(k, v) for k, v in test_unique_cols.items()][:5], len(test_unique_cols))","metadata":{"execution":{"iopub.status.busy":"2024-05-07T06:19:49.526144Z","iopub.execute_input":"2024-05-07T06:19:49.526678Z","iopub.status.idle":"2024-05-07T06:19:49.535042Z","shell.execute_reply.started":"2024-05-07T06:19:49.526638Z","shell.execute_reply":"2024-05-07T06:19:49.533809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save test unique features to json file\nwith open(\"test_unique_features.json\", \"w\") as f:\n    json.dump(test_unique_cols, f)","metadata":{"execution":{"iopub.status.busy":"2024-05-07T06:20:59.769676Z","iopub.execute_input":"2024-05-07T06:20:59.77026Z","iopub.status.idle":"2024-05-07T06:20:59.777518Z","shell.execute_reply.started":"2024-05-07T06:20:59.770222Z","shell.execute_reply":"2024-05-07T06:20:59.775848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train unique columns\ntrain_demo = pd.read_csv(\"/kaggle/input/leap-atmospheric-physics-ai-climsim/train.csv\", nrows=1)\ncolumns = train_demo.columns.to_list()\ncolumns.remove(\"sample_id\")\ntrain_chunks = pl.read_csv_batched(\"/kaggle/input/leap-atmospheric-physics-ai-climsim/train.csv\", raise_if_empty=False, batch_size=100_0000)\ntrain_unique_cols = {c: [] for c in columns}\ncount = 0\nwhile (chunk := train_chunks.next_batches(1)[0]) is not None:\n    count += 1\n    not_unique_cols = []\n    for col in tqdm(columns):\n        unique_flag, unique_value = check_unique(chunk[col])\n        if unique_flag:\n            train_unique_cols[col].append(unique_value)\n        else:\n            not_unique_cols.append(col)\n    columns = [x for x in columns if x not in not_unique_cols]","metadata":{"execution":{"iopub.status.busy":"2024-05-07T07:54:30.733378Z","iopub.execute_input":"2024-05-07T07:54:30.733933Z","iopub.status.idle":"2024-05-07T08:52:46.028611Z","shell.execute_reply.started":"2024-05-07T07:54:30.733897Z","shell.execute_reply":"2024-05-07T08:52:46.023874Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_unique_cols_save = {}\nfor c in columns:\n    values = train_unique_cols[c]\n    assert len(values) == count\n    if len(list(set(values))) == 1:\n        train_unique_cols_save[c] = values[0]\n        \nprint([(k, v) for k, v in train_unique_cols_save.items()][:5], len(train_unique_cols_save))","metadata":{"execution":{"iopub.status.busy":"2024-05-07T08:53:57.990347Z","iopub.execute_input":"2024-05-07T08:53:57.990854Z","iopub.status.idle":"2024-05-07T08:53:58.002913Z","shell.execute_reply.started":"2024-05-07T08:53:57.990817Z","shell.execute_reply":"2024-05-07T08:53:57.999839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save train unique columns to json file\nwith open(\"train_unique_columns.json\", \"w\") as f:\n    json.dump(train_unique_cols_save, f)","metadata":{"execution":{"iopub.status.busy":"2024-05-07T08:55:22.034303Z","iopub.execute_input":"2024-05-07T08:55:22.034871Z","iopub.status.idle":"2024-05-07T08:55:22.044038Z","shell.execute_reply.started":"2024-05-07T08:55:22.034824Z","shell.execute_reply":"2024-05-07T08:55:22.042555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_unique_targets = {}\ntrain_unique_features = {}\nfor k, v in train_unique_cols_save.items():\n    if k in df_weight.columns.to_list():\n        train_unique_targets[k] = v\n    if k in df_test.columns:\n        train_unique_features[k] = v\n        \nprint(len(train_unique_targets), len(train_unique_features))","metadata":{"execution":{"iopub.status.busy":"2024-05-07T09:00:05.295401Z","iopub.execute_input":"2024-05-07T09:00:05.296173Z","iopub.status.idle":"2024-05-07T09:00:05.316768Z","shell.execute_reply.started":"2024-05-07T09:00:05.296122Z","shell.execute_reply":"2024-05-07T09:00:05.315178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets_union = set(invalid_targets + list(train_unique_targets.keys()))\nprint(set(targets_union) == set(invalid_targets))\n# train_unique_targets is subset of invalid_targets","metadata":{"execution":{"iopub.status.busy":"2024-05-07T09:11:15.448355Z","iopub.execute_input":"2024-05-07T09:11:15.448969Z","iopub.status.idle":"2024-05-07T09:11:15.456584Z","shell.execute_reply.started":"2024-05-07T09:11:15.448924Z","shell.execute_reply":"2024-05-07T09:11:15.455245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(set(list(train_unique_features.keys())) == set(list(test_unique_cols.keys())))\n# train_unique_features equal to test_unique_cols","metadata":{"execution":{"iopub.status.busy":"2024-05-07T09:15:05.548434Z","iopub.execute_input":"2024-05-07T09:15:05.548946Z","iopub.status.idle":"2024-05-07T09:15:05.557557Z","shell.execute_reply.started":"2024-05-07T09:15:05.548908Z","shell.execute_reply":"2024-05-07T09:15:05.555862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols_value_check: list[bool] = []\nfor k, v in train_unique_features.items():\n    cols_value_check.append(v == test_unique_cols[k])\n    \nprint(all(cols_value_check))\n# value in train_unique_features equal to value in test_unique_cols","metadata":{"execution":{"iopub.status.busy":"2024-05-07T09:17:11.769848Z","iopub.execute_input":"2024-05-07T09:17:11.77031Z","iopub.status.idle":"2024-05-07T09:17:11.777775Z","shell.execute_reply.started":"2024-05-07T09:17:11.770275Z","shell.execute_reply":"2024-05-07T09:17:11.776281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reduce features\ndf_train = pd.read_csv(\"/kaggle/input/leap-atmospheric-physics-ai-climsim/train.csv\", nrows=1)\ndf_train[list(train_unique_features.keys())].to_csv(\"unique_features.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-07T09:19:49.946047Z","iopub.execute_input":"2024-05-07T09:19:49.946547Z","iopub.status.idle":"2024-05-07T09:19:50.04258Z","shell.execute_reply.started":"2024-05-07T09:19:49.946509Z","shell.execute_reply":"2024-05-07T09:19:50.041361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reduce targets\ndf_weight[invalid_targets].to_csv(\"invalid_targets.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-07T09:21:38.492415Z","iopub.execute_input":"2024-05-07T09:21:38.49285Z","iopub.status.idle":"2024-05-07T09:21:38.504217Z","shell.execute_reply.started":"2024-05-07T09:21:38.492818Z","shell.execute_reply":"2024-05-07T09:21:38.502834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Conclusion  \n## Static features  \n'[\"pbuf_CH4_27\", \"pbuf_CH4_28\", \"pbuf_CH4_29\", \"pbuf_CH4_30\", \"pbuf_CH4_31\", \"pbuf_CH4_32\", \"pbuf_CH4_33\", \"pbuf_CH4_34\", \"pbuf_CH4_35\", \"pbuf_CH4_36\", \"pbuf_CH4_37\", \"pbuf_CH4_38\", \"pbuf_CH4_39\", \"pbuf_CH4_40\", \"pbuf_CH4_41\", \"pbuf_CH4_42\", \"pbuf_CH4_43\", \"pbuf_CH4_44\", \"pbuf_CH4_45\", \"pbuf_CH4_46\", \"pbuf_CH4_47\", \"pbuf_CH4_48\", \"pbuf_CH4_49\", \"pbuf_CH4_50\", \"pbuf_CH4_51\", \"pbuf_CH4_52\", \"pbuf_CH4_53\", \"pbuf_CH4_54\", \"pbuf_CH4_55\", \"pbuf_CH4_56\", \"pbuf_CH4_57\", \"pbuf_CH4_58\", \"pbuf_CH4_59\", \"pbuf_N2O_27\", \"pbuf_N2O_28\", \"pbuf_N2O_29\", \"pbuf_N2O_30\", \"pbuf_N2O_31\", \"pbuf_N2O_32\", \"pbuf_N2O_33\", \"pbuf_N2O_34\", \"pbuf_N2O_35\", \"pbuf_N2O_36\", \"pbuf_N2O_37\", \"pbuf_N2O_38\", \"pbuf_N2O_39\", \"pbuf_N2O_40\", \"pbuf_N2O_41\", \"pbuf_N2O_42\", \"pbuf_N2O_43\", \"pbuf_N2O_44\", \"pbuf_N2O_45\", \"pbuf_N2O_46\", \"pbuf_N2O_47\", \"pbuf_N2O_48\", \"pbuf_N2O_49\", \"pbuf_N2O_50\", \"pbuf_N2O_51\", \"pbuf_N2O_52\", \"pbuf_N2O_53\", \"pbuf_N2O_54\", \"pbuf_N2O_55\", \"pbuf_N2O_56\", \"pbuf_N2O_57\", \"pbuf_N2O_58\", \"pbuf_N2O_59\"]'\n``` python\nimport json\nwith open(\"test_unique_features.json\", \"r\") as f:\n    static_features: dict[str, float] = json.load(f)\nstatic_features: list[str] = list(static_features.keys())\n```\n## Targets with zero weight\n'[\"ptend_q0001_0\", \"ptend_q0001_1\", \"ptend_q0001_2\", \"ptend_q0001_3\", \"ptend_q0001_4\", \"ptend_q0001_5\", \"ptend_q0001_6\", \"ptend_q0001_7\", \"ptend_q0001_8\", \"ptend_q0001_9\", \"ptend_q0001_10\", \"ptend_q0001_11\", \"ptend_q0002_0\", \"ptend_q0002_1\", \"ptend_q0002_2\", \"ptend_q0002_3\", \"ptend_q0002_4\", \"ptend_q0002_5\", \"ptend_q0002_6\", \"ptend_q0002_7\", \"ptend_q0002_8\", \"ptend_q0002_9\", \"ptend_q0002_10\", \"ptend_q0002_11\", \"ptend_q0002_12\", \"ptend_q0002_13\", \"ptend_q0002_14\", \"ptend_q0003_0\", \"ptend_q0003_1\", \"ptend_q0003_2\", \"ptend_q0003_3\", \"ptend_q0003_4\", \"ptend_q0003_5\", \"ptend_q0003_6\", \"ptend_q0003_7\", \"ptend_q0003_8\", \"ptend_q0003_9\", \"ptend_q0003_10\", \"ptend_q0003_11\", \"ptend_u_0\", \"ptend_u_1\", \"ptend_u_2\", \"ptend_u_3\", \"ptend_u_4\", \"ptend_u_5\", \"ptend_u_6\", \"ptend_u_7\", \"ptend_u_8\", \"ptend_u_9\", \"ptend_u_10\", \"ptend_u_11\", \"ptend_v_0\", \"ptend_v_1\", \"ptend_v_2\", \"ptend_v_3\", \"ptend_v_4\", \"ptend_v_5\", \"ptend_v_6\", \"ptend_v_7\", \"ptend_v_8\", \"ptend_v_9\", \"ptend_v_10\", \"ptend_v_11\"]'\n``` python\nimport json\nwith open(\"invalid_targets.json\", \"r\") as f:\n    invalid_targets: list[str] = json.load(f)\n```","metadata":{}}]}