diff --git a/anyway/parsers/schools_2025_empty_output.ipynb b/anyway/parsers/schools_2025_empty_output.ipynb index bf1c71dc..1eff5432 100644 --- a/anyway/parsers/schools_2025_empty_output.ipynb +++ b/anyway/parsers/schools_2025_empty_output.ipynb @@ -2191,22 +2191,33 @@ "metadata": {}, "source": [ "import io\n", - "#from anyway.parsers.cbs.s3.base import S3DataClass\n", - "# S3_BUCKET = \"dfc-anyway\"\n", - "# S3_SCHOOLS_FILE_PATH = \"schools_report/output\"\n", - "\n", - "# def load_school_file_from_s3(filename):\n", - "# s3_bucket = S3DataClass(s3_bucket_name=S3_BUCKET).s3_bucket\n", - "# object = s3_bucket.Object(os.path.join(S3_SCHOOLS_FILE_PATH,filename))\n", - "# data = json.loads(object.get()['Body'])\n", - "# return data\n", - "\n", - "# def upload_schools_file_to_s3(filename, data):\n", - "# s3_bucket = S3DataClass(s3_bucket_name=S3_BUCKET).s3_bucket\n", - "# with io.BytesIO() as json_buffer:\n", - "# json_buffer.write(json.dumps(data, ensure_ascii=False).encode('utf-8'))\n", - "# json_buffer.seek(0)\n", - "# s3_bucket.upload_fileobj(json_buffer, os.path.join(S3_SCHOOLS_FILE_PATH,filename))\n", + "from anyway.parsers.cbs.s3.base import S3DataClass\n", + "\n", + "S3_BUCKET = \"dfc-anyway\"\n", + "# Production API reads schools_report/output — keep generated files off that path.\n", + "S3_SCHOOLS_FILE_PATH = \"schools_report/output_2025\"\n", + "\n", + "\n", + "def s3_object_key(filename):\n", + " return f\"{S3_SCHOOLS_FILE_PATH}/{filename}\"\n", + "\n", + "\n", + "def upload_schools_file_to_s3(filename, data):\n", + " s3_bucket = S3DataClass(s3_bucket_name=S3_BUCKET).s3_bucket\n", + " with io.BytesIO() as json_buffer:\n", + " json_buffer.write(json.dumps(data, ensure_ascii=False).encode(\"utf-8\"))\n", + " json_buffer.seek(0)\n", + " key = s3_object_key(filename)\n", + " s3_bucket.upload_fileobj(json_buffer, key)\n", + " print(f\"Uploaded json -> s3://{S3_BUCKET}/{key}\")\n", + "\n", + "\n", + "def upload_local_file_to_s3(local_path, s3_filename=None):\n", + " s3_filename = s3_filename or os.path.basename(local_path)\n", + " key = s3_object_key(s3_filename)\n", + " S3DataClass(s3_bucket_name=S3_BUCKET).s3_bucket.upload_file(local_path, key)\n", + " print(f\"Uploaded {local_path} -> s3://{S3_BUCKET}/{key}\")\n", + "\n", "\n", "schools_data = json.load(open(os.path.join(MAIN_OUTPUT_DIR, \"schools_names.json\")))\n", "all_schools_df = pd.DataFrame(schools_data)" @@ -2298,7 +2309,7 @@ " total_dictionary_for_json_api[str(school_id)] = [r for r in injured_schools_per_year_list if r[\"school_id\"] == school_id]\n", "with open(os.path.join(schools_results_directory, \"injured_around_schools_api.json\"), 'w', encoding='utf-8') as f:\n", " json.dump(total_dictionary_for_json_api, f, ensure_ascii=False)\n", - "#upload_schools_file_to_s3(\"injured_around_schools_api.json\", total_dictionary_for_json_api)\n" + "upload_schools_file_to_s3(\"injured_around_schools_api.json\", total_dictionary_for_json_api)\n" ], "execution_count": null, "outputs": [] @@ -2336,9 +2347,9 @@ "total_dictionary_for_json_api = {}\n", "for school_id in all_schools_ids:\n", " total_dictionary_for_json_api[str(school_id)] = [r for r in injured_schools_per_month_list if r[\"school_id\"] == school_id]\n", - "#upload_schools_file_to_s3(\"injured_around_schools_months_graphs_data_api.json\", total_dictionary_for_json_api)\n", "with open(os.path.join(schools_results_directory, \"injured_around_schools_months_graphs_data_api.json\"), 'w', encoding='utf-8') as f:\n", - " json.dump(total_dictionary_for_json_api, f, ensure_ascii=False)\n" + " json.dump(total_dictionary_for_json_api, f, ensure_ascii=False)\n", + "upload_schools_file_to_s3(\"injured_around_schools_months_graphs_data_api.json\", total_dictionary_for_json_api)\n" ], "execution_count": null, "outputs": [] @@ -2376,9 +2387,26 @@ "total_dictionary_for_json_api = {}\n", "for school_id in all_schools_ids:\n", " total_dictionary_for_json_api[str(school_id)] = [r for r in injured_schools_per_sex_list if r[\"school_id\"] == school_id]\n", - "#upload_schools_file_to_s3(\"injured_around_schools_sex_graphs_data_api.json\", total_dictionary_for_json_api)\n", "with open(os.path.join(schools_results_directory, \"injured_around_schools_sex_graphs_data_api.json\"), 'w', encoding='utf-8') as f:\n", - " json.dump(total_dictionary_for_json_api, f, ensure_ascii=False)" + " json.dump(total_dictionary_for_json_api, f, ensure_ascii=False)\n", + "upload_schools_file_to_s3(\"injured_around_schools_sex_graphs_data_api.json\", total_dictionary_for_json_api)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### upload generated csvs to S3" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "for csv_path in sorted(glob.glob(os.path.join(schools_results_directory, \"*.csv\"))):\n", + " upload_local_file_to_s3(csv_path)" ], "execution_count": null, "outputs": []