diff --git a/data/scrape-share-food-program/.env_template b/data/scrape-share-food-program/.env_template new file mode 100644 index 00000000..1fff1826 --- /dev/null +++ b/data/scrape-share-food-program/.env_template @@ -0,0 +1,2 @@ +SUPABASE_URL= +SUPABASE_API_KEY= \ No newline at end of file diff --git a/data/scrape-share-food-program/README.md b/data/scrape-share-food-program/README.md index f9bb5dc8..5040c2a6 100644 --- a/data/scrape-share-food-program/README.md +++ b/data/scrape-share-food-program/README.md @@ -1,46 +1,35 @@ -# Share Food Program Scraping +# Share Food Program Sync -The Share Food Program can be found here: https://www.sharefoodprogram.org/ - -The site contains regularly-updated information about food resources in the Philadelphia area. This directory contains Python code for scraping this site. +Scrapes approved food distribution sites from the [Share Food Program](https://www.sharefoodprogram.org/) map API and upserts them into the Supabase `resources` table. All records written by this script use `creator = "phlask-share-food-program-sync"` — each run deletes those records then re-inserts fresh ones. ## Setup -### Install Python - -First, make sure to have Python 3.12+ installed. We also recommend using [PyCharm](https://www.jetbrains.com/pycharm/download) for Python development. - -### Create a Virtual Environment and Install Dependencies - -Inside of this directory, run the following commands: - ```bash python -m venv .venv -# If on Mac/Linux -source .venv/bin/activate -# If on Windows -.venv\Scripts\activate +source .venv/bin/activate # Windows: .venv\Scripts\activate pip install -r requirements.txt ``` -### Add Firebase Credentials +Create a `.env` file in this directory: -To run the scraper and upload the data to Firebase, you will need to add your Firebase credentials to this folder. Message us in the #phlask_data channel on Slack to get access. +```env +SUPABASE_URL=https://your-project.supabase.co +SUPABASE_API_KEY=your-service-role-key +``` -### Run the Scraper +Message us in `#phlask_data` on Slack to get the credentials. -To run the scraper, use the following command, making sure to set the URL below to the correct URL for your Firebase instance. +## Usage +**Sync to Supabase:** ```bash -python scrape_share_food_program.py https://phlask-share-food-test.firebaseio.com/ +python scrape_share_food_program.py ``` -You should see output like the following: - +**Debug locally (no Supabase required):** +```bash +python scrape_share_food_program.py --csv # writes resources.csv +python scrape_share_food_program.py --csv out.csv # custom filename ``` -Got 169 new resources from the scraped resource -Using DB URL: https://phlask-share-food-test.firebaseio.com/ -Loaded PHLASK DB reference with 819 resources -Removed 169 existing scraped resources from the DB -We now have 819 total resources in the DB -``` \ No newline at end of file + +The CSV serializes JSONB fields (`source`, `verification`, `food`) as JSON strings so the output is inspectable without a database connection. diff --git a/data/scrape-share-food-program/requirements.txt b/data/scrape-share-food-program/requirements.txt index af6345f3..9978f320 100644 --- a/data/scrape-share-food-program/requirements.txt +++ b/data/scrape-share-food-program/requirements.txt @@ -1,81 +1,4 @@ -asttokens==3.0.0 -attrs==24.3.0 -backcall==0.2.0 beautifulsoup4==4.12.3 -bleach==6.2.0 -CacheControl==0.14.1 -cachetools==5.5.0 -certifi==2024.8.30 -cffi==1.17.1 -charset-normalizer==3.4.0 -colorama==0.4.6 -cryptography==46.0.6 -decorator==5.1.1 -defusedxml==0.7.1 -docopt==0.6.2 -executing==2.1.0 -fastjsonschema==2.21.1 -firebase-admin==6.6.0 -google-api-core==2.23.0 -google-api-python-client==2.154.0 -google-auth==2.36.0 -google-auth-httplib2==0.2.0 -google-cloud-core==2.4.1 -google-cloud-firestore==2.19.0 -google-cloud-storage==2.18.2 -google-crc32c==1.6.0 -google-resumable-media==2.7.2 -googleapis-common-protos==1.66.0 -grpcio==1.68.0 -grpcio-status==1.68.0 -httplib2==0.22.0 -idna==3.10 -ipython==8.12.3 -jedi==0.19.2 -Jinja2==3.1.6 -jsonschema==4.23.0 -jsonschema-specifications==2024.10.1 -jupyter_client==8.6.3 -jupyter_core==5.7.2 -jupyterlab_pygments==0.3.0 -MarkupSafe==3.0.2 -matplotlib-inline==0.1.7 -mistune==3.1.0 -msgpack==1.1.0 -nbclient==0.10.2 -nbconvert==7.17.0 -nbformat==5.10.4 -packaging==24.2 -pandocfilters==1.5.1 -parso==0.8.4 -pickleshare==0.7.5 -pipreqs==0.5.0 -platformdirs==4.3.6 -prompt_toolkit==3.0.48 -proto-plus==1.25.0 -protobuf==6.33.5 -pure_eval==0.2.3 -pyasn1==0.6.3 -pyasn1_modules==0.4.1 -pycparser==2.22 -Pygments==2.20.0 -PyJWT==2.12.0 -pyparsing==3.2.0 -python-dateutil==2.9.0.post0 -pywin32==308 -pyzmq==26.2.0 -referencing==0.35.1 +python-dotenv>=1.0.0 requests==2.33.0 -rpds-py==0.22.3 -rsa==4.9 -six==1.17.0 -soupsieve==2.6 -stack-data==0.6.3 -tinycss2==1.4.0 -tornado==6.5.5 -traitlets==5.14.3 -uritemplate==4.1.1 -urllib3==2.6.3 -wcwidth==0.2.13 -webencodings==0.5.1 -yarg==0.1.9 +supabase>=2.3.0 diff --git a/data/scrape-share-food-program/scrape_share_food_program.py b/data/scrape-share-food-program/scrape_share_food_program.py index 1bdecefb..292965d9 100644 --- a/data/scrape-share-food-program/scrape_share_food_program.py +++ b/data/scrape-share-food-program/scrape_share_food_program.py @@ -1,9 +1,26 @@ -import requests +import argparse +import csv import datetime -from bs4 import BeautifulSoup +import json import os -import sys -import uuid +import requests +from bs4 import BeautifulSoup +from supabase import create_client, Client +from dotenv import load_dotenv + +load_dotenv() + +# config +SUPABASE_URL = os.environ.get("SUPABASE_URL", "") +SUPABASE_KEY = os.environ.get("SUPABASE_API_KEY", "") +TABLE_NAME = "resources" + +CREATOR = "phlask-share-food-program-sync" +SOURCE_URL = ( + "https://www.sharefoodprogram.org/wp-json/wpgmza/v1/features/" + "base64eJyrVkrLzClJLVKyUqqOUcpNLIjPTIlRsopRMo5R0gEJFGeUFni6" + "FAPFomOBAsmlxSX5uW6ZqTkpELFapVoABaMWvA" +) def extract_street_from_address(address): @@ -11,96 +28,159 @@ def extract_street_from_address(address): def convert_html_to_text(html): - # Take the HTML and convert it into a plaintext string soup = BeautifulSoup(html, "html.parser") for elem in soup.find_all(["a", "p", "div", "h3", "br"]): elem.replace_with(elem.text + "\n") - plain_text = soup.get_text(separator="\n") - return plain_text - - -url = "https://www.sharefoodprogram.org/wp-json/wpgmza/v1/features/base64eJyrVkrLzClJLVKyUqqOUcpNLIjPTIlRsopRMo5R0gEJFGeUFni6FAPFomOBAsmlxSX5uW6ZqTkpELFapVoABaMWvA" - -response = requests.get(url, headers={ - 'User-Agent': 'PostmanRuntime/7.43.0' -}) - -data = response.json() + return soup.get_text(separator="\n") -new_phlask_data = {} -for marker in data['markers']: +def fetch_markers() -> list[dict]: + response = requests.get(SOURCE_URL, headers={"User-Agent": "PostmanRuntime/7.43.0"}) + response.raise_for_status() + return [m for m in response.json()["markers"] if m["approved"] == "1"] - if marker['approved'] != "1": - continue - current_timestamp = datetime.datetime.now().isoformat() - - new_phlask_resource = { - "address": extract_street_from_address(marker['address']), - "city": "Philadelphia", - "creator": "phlask", - "date_created": current_timestamp, - "description": convert_html_to_text(marker['description']), - "entry_type": "UNSURE", - "last_modified": current_timestamp, - "last_modifier": "phlask", - "latitude": float(marker['lat']), - "longitude": float(marker['lng']), - "name": marker['title'], - "resource_type": "FOOD", +def marker_to_resource(marker: dict) -> dict: + now = datetime.datetime.now(tz=datetime.timezone.utc).isoformat(timespec="milliseconds") + return { + "version": 1, + "creator": CREATOR, + "last_modifier": CREATOR, + "date_created": now, + "last_modified": now, "source": { - "type": "WEB_SCRAPE", - "url": url, - "logo_url": "https://www.sharefoodprogram.org/wp-content/themes/sharefood-theme/images/svg/share-food-program-logo.svg" + "type": "WEB_SCRAPE", + "url": SOURCE_URL, + "logo_url": "https://www.sharefoodprogram.org/wp-content/themes/sharefood-theme/images/svg/share-food-program-logo.svg", }, - "state": "PA", - "status": "OPERATIONAL", "verification": { - "last_modified": current_timestamp, - "last_modifier": "phlask", - "verified": True + "verified": True, + "last_modified": now, + "last_modifier": CREATOR, }, - "version": 1, + "resource_type": "FOOD", + "status": "OPERATIONAL", + "entry_type": "UNSURE", + "name": marker["title"], + "description": convert_html_to_text(marker["description"]), + "address": extract_street_from_address(marker["address"]), + "city": "Philadelphia", + "state": "PA", + "zip_code": "19104", # TODO: derive per-marker via geocoding + "latitude": float(marker["lat"]), + "longitude": float(marker["lng"]), "food": { - "food_type": [], + "food_type": [], "distribution_type": [], - "organization_type": [] + "organization_type": [], }, - "zip_code": "19104" # TODO: Change to zip code from lookup using geocoding + "hours": None, + "images": None, + "guidelines": None, + "water": None, + "forage": None, + "bathroom": None, } - new_phlask_data[str(uuid.uuid4())] = new_phlask_resource - -print(f"Got {len(new_phlask_data)} new resources from the scraped resource") - -cert_path = os.path.abspath("firebase_cert.json") -os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = cert_path - -# Grab the DB URL from the python run command -DB_URL = sys.argv[1] -print(f"Using DB URL: {DB_URL}") -if not DB_URL: - raise ValueError("DB_URL not set, make sure to append this after your python run command") - -from firebase_admin import initialize_app, db -default_app = initialize_app() -ref = db.reference(url=DB_URL) -existing_phlask_data: dict = ref.get() -print(f"Loaded PHLASK DB reference with {len(existing_phlask_data)} resources") - -# Remove the existing resources from this scraped data so we don't have duplicates -before_len = len(existing_phlask_data) -existing_phlask_data = {resource_name: resource for resource_name, resource in existing_phlask_data.items() if resource['source'].get('url') != url} -after_len = len(existing_phlask_data) -print(f"Removed {before_len - after_len} existing scraped resources from the DB") - -# Add the new resources -existing_phlask_data.update(new_phlask_data) - -# Set the new data in firebase -ref.set(existing_phlask_data) -# Verify that the new data was pushed -new_data_test = ref.get() -print(f"We now have {len(new_data_test)} total resources in the DB") \ No newline at end of file +# Supabase helpers + +def get_supabase_client() -> Client: + if not SUPABASE_URL or not SUPABASE_KEY: + raise EnvironmentError( + "SUPABASE_URL and SUPABASE_API_KEY must be set. " + "Use --csv to output locally instead." + ) + return create_client(SUPABASE_URL, SUPABASE_KEY) + + +def delete_by_creator(client: Client) -> None: + # Warn about any records from the same source URL with a mismatched creator — + # these are stale rows from a previous naming convention that won't be cleaned up. + mismatched = ( + client.table(TABLE_NAME) + .select("id, creator") + .filter("source->>url", "eq", SOURCE_URL) + .neq("creator", CREATOR) + .execute() + ) + if mismatched.data: + print(f" [debug] {len(mismatched.data)} record(s) match source URL but have unexpected creator:") + for row in mismatched.data: + print(f" id={row['id']} creator={row['creator']!r}") + + result = client.table(TABLE_NAME).delete().eq("creator", CREATOR).execute() + count = len(result.data) if result.data else 0 + print(f"Deleted {count} existing record(s) with creator='{CREATOR}'.") + + +def insert_resources(client: Client, resources: list[dict]) -> None: + if not resources: + print("No resources to insert.") + return + client.table(TABLE_NAME).insert(resources).execute() + print(f"Inserted {len(resources)} resource(s).") + + +# CSV for debugging/local + +def save_csv(resources: list[dict], path: str) -> None: + if not resources: + print("No resources found — CSV not written.") + return + + JSONB_FIELDS = {"source", "verification", "food"} + fieldnames = [ + "name", "resource_type", "status", "entry_type", + "address", "city", "state", "zip_code", "latitude", "longitude", + "description", "source", "verification", "food", + "creator", "last_modifier", "version", + ] + + def serialize(key, val): + if val is None: + return "" + if key in JSONB_FIELDS: + return json.dumps(val) + return val + + with open(path, "w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore") + writer.writeheader() + for r in resources: + writer.writerow({k: serialize(k, r.get(k)) for k in fieldnames}) + + print(f"Saved {len(resources)} resource(s) → {path}") + + +# CLI + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Sync Share Food Program data to Supabase resources table." + ) + parser.add_argument( + "--csv", + nargs="?", + const="resources.csv", + default=None, + metavar="FILE", + help="Output to CSV instead of Supabase (default filename: resources.csv).", + ) + args = parser.parse_args() + + print("Fetching markers from Share Food Program...") + markers = fetch_markers() + print(f"Found {len(markers)} approved marker(s). Normalizing...") + + resources = [marker_to_resource(m) for m in markers] + print(f"Normalized {len(resources)} resource(s).") + + if args.csv: + save_csv(resources, args.csv) + else: + supabase = get_supabase_client() + delete_by_creator(supabase) + insert_resources(supabase, resources) + + print("Done.")