{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":56537,"databundleVersionId":8015876,"sourceType":"competition"},{"sourceId":8175194,"sourceType":"datasetVersion","datasetId":4838662},{"sourceId":173007160,"sourceType":"kernelVersion"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nimport polars.selectors as cs\nfrom gc import collect\nimport os\n\nprint(f\"\\n---> Inspired from https://www.kaggle.com/code/asarvazyan/leap-predict-the-mean\\n\")\n\nsample_submission = \\\npl.scan_parquet(\"/kaggle/input/climsim2024starterv1/sample_submission.parquet\")","metadata":{"execution":{"iopub.status.busy":"2024-04-20T13:46:21.320974Z","iopub.execute_input":"2024-04-20T13:46:21.321445Z","iopub.status.idle":"2024-04-20T13:46:21.332352Z","shell.execute_reply.started":"2024-04-20T13:46:21.321408Z","shell.execute_reply":"2024-04-20T13:46:21.331058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **DATA COLLATION**","metadata":{}},{"cell_type":"code","source":"%%time \n\n# Collating data:-\nroot        = \"/kaggle/input/climsim2024starterv1\"\ntargets     = [x for x in sample_submission.columns if x not in [\"sample_id\"]]\nall_means   = pl.DataFrame(schema = targets)\nall_folders = [d for d in os.listdir(root) if d != \"sample_submission.parquet\"]\n\n# Collating the data:-\nfor folder in all_folders:\n    print(f\"\\nFolder - {folder}\\n\");\n    \n    for batch_id in range(0, 500,1):\n        mypath    = os.path.join(root, folder, f\"Train_Batch{batch_id}.parquet\")\n        df        = pl.scan_parquet(mypath).select(targets).mean()\n        all_means = pl.concat([all_means, df.collect()], how = \"vertical_relaxed\")\n        \n        if batch_id % 10 ==0:\n            print(f\"Train_Batch{batch_id} | All means shape = {all_means.shape}\")\n        del df\n        \n# Collating from my kernel - additional 2.5 million records:-\nroot     = f\"/kaggle/input/climsim2024-starterdata-v1\"\nallfiles = os.listdir(root)\nprint(\"\\nCollating from my kernel\\n\")\n\nfor batch_id in range(0, 500,1):\n    mypath    = os.path.join(root, f\"Train_Batch{batch_id}.parquet\")\n    df        = pl.scan_parquet(mypath).select(targets).mean()\n    all_means = pl.concat([all_means, df.collect()], how = \"vertical_relaxed\")\n    \n    if batch_id % 10 ==0:\n            print(f\"Train_Batch{batch_id} | All means shape = {all_means.shape}\")\n    del df","metadata":{"execution":{"iopub.status.busy":"2024-04-20T14:01:03.923942Z","iopub.execute_input":"2024-04-20T14:01:03.924669Z","iopub.status.idle":"2024-04-20T14:02:07.795127Z","shell.execute_reply.started":"2024-04-20T14:01:03.924614Z","shell.execute_reply":"2024-04-20T14:02:07.793492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **SUBMISSION**","metadata":{}},{"cell_type":"code","source":"%%time \n\nsample_submission = \\\npd.read_parquet(\"/kaggle/input/climsim2024starterv1/sample_submission.parquet\")\nfinal_mean = all_means.mean().to_pandas().transpose().squeeze()\n\nfor col in final_mean.keys():\n    sample_submission[col] = sample_submission[col] * final_mean[col]\n    \ndisplay(sample_submission.head(10).\\\n        style.\\\n        set_caption(f\"\\nSubmission file\\n\").\\\n        format(precision = 3)\n       )\n\nsample_submission.to_parquet(\"submission.parquet\")\ncollect();","metadata":{"execution":{"iopub.status.busy":"2024-04-20T14:09:23.487367Z","iopub.execute_input":"2024-04-20T14:09:23.487796Z","iopub.status.idle":"2024-04-20T14:09:27.907025Z","shell.execute_reply.started":"2024-04-20T14:09:23.487765Z","shell.execute_reply":"2024-04-20T14:09:27.905678Z"},"trusted":true},"execution_count":null,"outputs":[]}]}