diff --git a/.gitignore b/.gitignore index d556fe7..26f6de4 100644 --- a/.gitignore +++ b/.gitignore @@ -81,9 +81,6 @@ target/ profile_default/ ipython_config.py -# pyenv -.python-version - # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..6324d40 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.14 diff --git a/generate.py b/generate.py index 5efb930..834c801 100644 --- a/generate.py +++ b/generate.py @@ -1,12 +1,16 @@ import requests as rq -import csv import datetime -import numpy as np -import matplotlib as mpl +import polars as pl +import seaborn as sns +from zoneinfo import ZoneInfo import matplotlib.pyplot as plt +from matplotlib.patches import Patch +from matplotlib.lines import Line2D import matplotlib.dates as mdates -from pipeline import * +import matplotlib.ticker as ticker +now = datetime.datetime.now(ZoneInfo("America/Vancouver")) +current_year = now.year params = { "DataSet": "SGWL.Working@OW283", @@ -23,61 +27,86 @@ url="https://aqrt.nrs.gov.bc.ca/Export/DataSetToken", params=params ).json()["Token"] params["Token"] = token -response = rq.get(url="https://aqrt.nrs.gov.bc.ca/Export/DataSet", params=params) -reader = csv.reader(response.text.splitlines()) - -# with open("data/dataset.csv") as data: -parsed = filter(lambda x: x, map(lambda x: clean_and_process(x[0], x[1]), reader)) -split = year_splitter(parsed) -years = list(map(lambda x: np.transpose(unify_year(x)), split)) +response = rq.get(url="https://aqrt.nrs.gov.bc.ca/Export/DataSet", params=params) +df = pl.read_csv( + response.content, + try_parse_dates=True, + skip_lines=5, + new_columns=["timestamp", "level"], +) +df = df.drop_nans() +df = df.group_by_dynamic("timestamp", every="1d").agg(pl.col("level").mean()) +df = df.with_columns( + (pl.col("level") * -3.28), (pl.col("timestamp").dt.year()).alias("year") +) +df = df.remove( + (pl.col("timestamp").dt.month() == 2) & (pl.col("timestamp").dt.day() == 29) +) +df = df.with_columns( + (pl.col("timestamp").dt.replace(year=current_year)), + (pl.col("year") == current_year).alias("current_year"), +) +df = df.remove((pl.col("year") < 2004)) + +previous_years = df.filter(~pl.col("current_year")) +redline = df.filter(pl.col("current_year")) + +ordinal_today = now.date().replace(year=1).toordinal() +historical_average: float = -( + previous_years.filter( + (pl.col("timestamp").dt.ordinal_day() - ordinal_today).abs() <= 7 + ) + .group_by("year") + .agg(pl.col("level").mean()) + .mean()["level"] + .item() +) +current_level: float = -redline.row(-1, named=True)["level"] fig, ax = plt.subplots() fig.set_size_inches(14, 10) - +sns.lineplot( + ax=ax, + data=previous_years, + x="timestamp", + y="level", + color="teal", + errorbar=("pi", 90), + linewidth=0, +) +sns.lineplot(data=redline, x="timestamp", y="level", ax=ax, color="blue") +handles = [ + Patch(facecolor="teal", alpha=0.3, label="historical range"), + Line2D([0], [0], color="blue", label=f"{current_year}"), +] + +ax.legend(handles=handles) +ax.set_xlabel("") ax.set_ylabel("feet below the surface") ax.set_title("North Pender Island Water Table Status") -ax.set_xlim(years[-1][0][0], years[0][0][-1]) +ax.set_xlim(datetime.date(current_year, 1, 1), datetime.date(current_year, 12, 31)) # type: ignore[arg-type] -# fancy automatic date labels along x axis locator = mdates.AutoDateLocator(minticks=3, maxticks=20) formatter = mdates.ConciseDateFormatter(locator) ax.xaxis.set_major_locator(locator) ax.xaxis.set_major_formatter(formatter) - -# chosen for linear perception and contrast with red -colourmap = mpl.colormaps['viridis'] # type: ignore - -# graphing each year with its own label and colour along the viridis colourmap -year_plots = [] -current_year = datetime.today().year -for i, year in enumerate(years[7:-1]): - year_as_percentage = i / (current_year - 1 - 2010) - (year_plot,) = ax.plot( - year[0][:-40], - rolling_mean(year[1], 40)[:-40], - color=colourmap(1 - year_as_percentage), - label=i + 2010, - ) - year_plots.append(year_plot) - -# current year -ax.plot(years[-1][0], years[-1][1], color="red", label=current_year) - -ax.legend() - +ax.yaxis.set_major_locator(ticker.MaxNLocator(integer=True)) fig.savefig("www/output.svg") fig.set_size_inches(7, 5) fig.savefig("www/output_small.svg") -current_level = abs(years[-1][-1][-1]) -historical = abs(historical_past_two_weeks(years)) +def rough_date() -> str: + month = now.strftime("%B") + prefix = "early " if now.day < 10 else "mid-" if now.day < 20 else "late " + return prefix + month with open("index.html") as index: - index = index.read().replace("XX", ("%2.1f" % current_level), 1) - index = index.replace("XX", ("%2.1f" % historical), 1) + index = index.read().replace("XX", now.strftime("%Y-%m-%d, %H:%M PT"), 1) + index = index.replace("XX", ("%2.1f" % current_level), 1) + index = index.replace("XX", ("%2.1f" % historical_average), 1) index = index.replace("this time of year", rough_date()) with open("www/index.html", "w") as new_index: new_index.write(index) @@ -96,7 +125,7 @@ """, "", ) - .replace('viewBox="0 0 1008 720"', 'viewBox="80 50 915 670"') + .replace('viewBox="0 0 1008 720"', 'viewBox="80 60 840 620"') .replace('height="720pt"', "") .replace('width="1008pt"', "") ) @@ -116,7 +145,7 @@ """, "", ) - .replace('viewBox="0 0 504 360"', 'viewBox="15 20 490 350"') + .replace('viewBox="0 0 504 360"', 'viewBox="15 20 445 330"') .replace('height="360pt"', "") .replace('width="504pt"', "") ) diff --git a/index.html b/index.html index ec6d4e0..8fb7723 100644 --- a/index.html +++ b/index.html @@ -14,7 +14,7 @@

North Pender Island Water Table

-

+

DATA UPDATED XX

The water level is currently

XX

@@ -25,15 +25,11 @@

-

Time range: 2010 - 2021
- Smoothing: 48 hours
- AI prediction: no
- Last updated: 30 minutes ago