From 9d595ca8dd68d2a3cdeef0a5dcb32126d2d8f1f8 Mon Sep 17 00:00:00 2001 From: Tom Ward Date: Thu, 4 Jun 2026 16:43:57 +0100 Subject: [PATCH] Batch the order & copy from prescribing_norm_tmp --- openprescribing/data/ingestors/prescribing.py | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/openprescribing/data/ingestors/prescribing.py b/openprescribing/data/ingestors/prescribing.py index f068c41d..73d89e2b 100644 --- a/openprescribing/data/ingestors/prescribing.py +++ b/openprescribing/data/ingestors/prescribing.py @@ -281,10 +281,30 @@ def ingest_sources(conn): params=[bnf_start, bnf_end], ) - conn.sql( - "INSERT INTO prescribing_norm SELECT * FROM prescribing_norm_tmp " - " ORDER BY presentation_id, date_id, practice_id" - ) + for bnf_start, bnf_end in get_bnf_code_ranges( + conn, batch_size=BNF_RANGE_BATCH_SIZE + ): + log.info(f"Building `prescribing_norm` table: {bnf_start} -> {bnf_end}") + conn.sql( + # Copy all fields /except/ bnf_code from prescribing_norm_tmp + """\ + INSERT INTO prescribing_norm + SELECT + presentation_id, + date_id, + practice_id, + quantity_value, + items, + quantity, + net_cost, + actual_cost + FROM prescribing_norm_tmp + WHERE prescribing_norm_tmp.bnf_code >= ? AND prescribing_norm_tmp.bnf_code < ? + ORDER BY presentation_id, date_id, practice_id + """, + params=[bnf_start, bnf_end], + ) + conn.sql("DROP TABLE prescribing_norm_tmp ") log.info(f"Ingested {count_table(conn, 'prescribing_norm'):,} prescribing rows") @@ -412,6 +432,7 @@ def sql_for_prescribing_normalised(): return """\ SELECT presentation.id AS presentation_id, + presentation.bnf_code AS bnf_code, date.id AS date_id, practice.id AS practice_id, prescribing_source.quantity_value,