From 1fec1e55883e57f5b2aa0a4f965295dd027a8cf0 Mon Sep 17 00:00:00 2001 From: xrendan Date: Mon, 31 Aug 2026 08:47:33 -0600 Subject: [PATCH 1/3] Add national municipal financial statement API --- ...nicipal_financial_statements_controller.rb | 402 +++++++++++ ...ract_municipal_financial_statements_job.rb | 90 +++ app/models/warehouse/census_profile.rb | 12 + .../warehouse/census_profile_importer.rb | 92 +++ .../financial_statement_extraction.rb | 44 +- .../candidate_set.rb | 133 ++++ .../candidate_window.rb | 28 + .../coverage_audit.rb | 170 +++++ .../detailed_pipeline.rb | 312 +++++++++ .../detailed_response_schema.rb | 17 + .../extractor.rb | 125 +++- .../failed_candidate_filter.rb | 138 ++++ .../fallback_pipeline.rb | 16 + .../number_parser.rb | 83 ++- .../ocr_text_cache.rb | 174 +++++ .../page_locator.rb | 242 ++++++- .../pipeline.rb | 91 ++- .../prairie_failed_candidate_filter.rb | 3 + .../processor.rb | 105 +++ .../quebec_form_pipeline.rb | 438 ++++++++++++ .../quebec_form_processor.rb | 139 ++++ .../response_schema.rb | 3 + .../reviewer.rb | 263 ++++++++ .../saskatchewan_form_pipeline.rb | 545 +++++++++++++++ .../saskatchewan_form_processor.rb | 5 + .../scale_detector.rb | 16 + .../stored_headline_pipeline.rb | 34 + .../validator.rb | 139 +++- .../visual_evidence_response_schema.rb | 15 + .../visual_evidence_reviewer.rb | 145 ++++ .../financial_statement_line_item.rb | 18 + config/routes.rb | 4 + ...1_create_financial_statement_line_items.rb | 33 + .../20260829000002_create_census_profiles.rb | 27 + ...financial_statement_extraction_identity.rb | 9 + ...epeated_financial_statement_line_labels.rb | 11 + ...ire_approved_financial_statement_checks.rb | 8 + ...ire_approved_financial_statement_review.rb | 8 + ...re_completed_financial_statement_checks.rb | 9 + db/structure.sql | 204 +++++- docs/plans/municipal_budget_acquisition.md | 91 +++ ...nicipal_financial_statements_deployment.md | 124 ++++ .../municipal_zero_publication_remediation.md | 52 ++ .../national_municipal_release_2026-08-27.md | 2 + script/apply_prairie_parser_upgrade_audit.rb | 59 ++ ...udit_financial_statement_numeric_values.rb | 50 ++ script/audit_financial_statement_scales.rb | 57 ++ ...municipal_financial_extraction_coverage.rb | 23 + script/audit_prairie_parser_upgrade.rb | 93 +++ ...enqueue_municipal_financial_extractions.rb | 42 ++ script/import_census_profile_population.rb | 17 + script/import_municipal_financial_pilot.rb | 72 ++ script/process_municipal_financial_details.rb | 97 +++ .../process_municipal_financial_statements.rb | 102 +++ script/process_quebec_financial_forms.rb | 36 + .../process_saskatchewan_financial_forms.rb | 58 ++ ...evalidate_municipal_financial_headlines.rb | 49 ++ ...xtracted_municipal_financial_statements.rb | 132 ++++ script/run_atlantic_financial_handoff.zsh | 105 +++ .../run_national_financial_finalization.zsh | 246 +++++++ script/run_ns_bc_financial_handoff.zsh | 111 ++++ script/run_on_final_financial_handoff.zsh | 46 ++ script/run_prairie_v3_financial_handoff.zsh | 217 ++++++ script/run_qc_generic_fallback_handoff.zsh | 110 +++ script/run_territory_financial_handoff.zsh | 57 ++ script/sanitize_municipal_report_batch.rb | 2 +- ...al_financial_statements_controller_test.rb | 401 +++++++++++ ...municipal_financial_statements_job_test.rb | 91 +++ .../warehouse/census_profile_importer_test.rb | 32 + .../candidate_set_test.rb | 79 +++ .../candidate_window_test.rb | 29 + .../detailed_pipeline_test.rb | 353 ++++++++++ .../number_parser_test.rb | 114 +++- .../ocr_text_cache_test.rb | 123 ++++ .../page_locator_test.rb | 210 ++++++ .../pipeline_test.rb | 81 +++ .../prairie_failed_candidate_filter_test.rb | 246 +++++++ .../quebec_form_pipeline_test.rb | 111 ++++ .../reviewer_test.rb | 408 ++++++++++++ .../saskatchewan_form_pipeline_test.rb | 624 ++++++++++++++++++ .../scale_detector_test.rb | 19 + .../stored_headline_pipeline_test.rb | 65 ++ .../validator_test.rb | 137 ++++ .../financial_statement_extraction_test.rb | 194 +++++- ...ipal_financial_extraction_coverage_test.rb | 156 +++++ ...ess_municipal_financial_statements_test.rb | 37 ++ .../sanitize_municipal_report_batch_test.rb | 37 ++ 87 files changed, 9875 insertions(+), 72 deletions(-) create mode 100644 app/controllers/api/v1/warehouse/municipal_financial_statements_controller.rb create mode 100644 app/jobs/warehouse/extract_municipal_financial_statements_job.rb create mode 100644 app/models/warehouse/census_profile.rb create mode 100644 app/models/warehouse/census_profile_importer.rb create mode 100644 app/models/warehouse/financial_statement_extraction/candidate_set.rb create mode 100644 app/models/warehouse/financial_statement_extraction/candidate_window.rb create mode 100644 app/models/warehouse/financial_statement_extraction/coverage_audit.rb create mode 100644 app/models/warehouse/financial_statement_extraction/detailed_pipeline.rb create mode 100644 app/models/warehouse/financial_statement_extraction/detailed_response_schema.rb create mode 100644 app/models/warehouse/financial_statement_extraction/failed_candidate_filter.rb create mode 100644 app/models/warehouse/financial_statement_extraction/fallback_pipeline.rb create mode 100644 app/models/warehouse/financial_statement_extraction/ocr_text_cache.rb create mode 100644 app/models/warehouse/financial_statement_extraction/prairie_failed_candidate_filter.rb create mode 100644 app/models/warehouse/financial_statement_extraction/processor.rb create mode 100644 app/models/warehouse/financial_statement_extraction/quebec_form_pipeline.rb create mode 100644 app/models/warehouse/financial_statement_extraction/quebec_form_processor.rb create mode 100644 app/models/warehouse/financial_statement_extraction/reviewer.rb create mode 100644 app/models/warehouse/financial_statement_extraction/saskatchewan_form_pipeline.rb create mode 100644 app/models/warehouse/financial_statement_extraction/saskatchewan_form_processor.rb create mode 100644 app/models/warehouse/financial_statement_extraction/scale_detector.rb create mode 100644 app/models/warehouse/financial_statement_extraction/stored_headline_pipeline.rb create mode 100644 app/models/warehouse/financial_statement_extraction/visual_evidence_response_schema.rb create mode 100644 app/models/warehouse/financial_statement_extraction/visual_evidence_reviewer.rb create mode 100644 app/models/warehouse/financial_statement_line_item.rb create mode 100644 db/migrate/20260829000001_create_financial_statement_line_items.rb create mode 100644 db/migrate/20260829000002_create_census_profiles.rb create mode 100644 db/migrate/20260829000003_add_year_to_financial_statement_extraction_identity.rb create mode 100644 db/migrate/20260829000004_allow_repeated_financial_statement_line_labels.rb create mode 100644 db/migrate/20260829000005_require_approved_financial_statement_checks.rb create mode 100644 db/migrate/20260829000006_require_approved_financial_statement_review.rb create mode 100644 db/migrate/20260829000007_require_completed_financial_statement_checks.rb create mode 100644 docs/plans/municipal_budget_acquisition.md create mode 100644 docs/plans/municipal_financial_statements_deployment.md create mode 100644 docs/plans/municipal_zero_publication_remediation.md create mode 100644 script/apply_prairie_parser_upgrade_audit.rb create mode 100644 script/audit_financial_statement_numeric_values.rb create mode 100755 script/audit_financial_statement_scales.rb create mode 100644 script/audit_municipal_financial_extraction_coverage.rb create mode 100644 script/audit_prairie_parser_upgrade.rb create mode 100644 script/enqueue_municipal_financial_extractions.rb create mode 100644 script/import_census_profile_population.rb create mode 100644 script/import_municipal_financial_pilot.rb create mode 100644 script/process_municipal_financial_details.rb create mode 100644 script/process_municipal_financial_statements.rb create mode 100755 script/process_quebec_financial_forms.rb create mode 100755 script/process_saskatchewan_financial_forms.rb create mode 100644 script/revalidate_municipal_financial_headlines.rb create mode 100644 script/review_extracted_municipal_financial_statements.rb create mode 100644 script/run_atlantic_financial_handoff.zsh create mode 100644 script/run_national_financial_finalization.zsh create mode 100644 script/run_ns_bc_financial_handoff.zsh create mode 100644 script/run_on_final_financial_handoff.zsh create mode 100644 script/run_prairie_v3_financial_handoff.zsh create mode 100644 script/run_qc_generic_fallback_handoff.zsh create mode 100644 script/run_territory_financial_handoff.zsh create mode 100644 test/controllers/api/v1/warehouse/municipal_financial_statements_controller_test.rb create mode 100644 test/jobs/warehouse/extract_municipal_financial_statements_job_test.rb create mode 100644 test/models/warehouse/census_profile_importer_test.rb create mode 100644 test/models/warehouse/financial_statement_extraction/candidate_set_test.rb create mode 100644 test/models/warehouse/financial_statement_extraction/candidate_window_test.rb create mode 100644 test/models/warehouse/financial_statement_extraction/detailed_pipeline_test.rb create mode 100644 test/models/warehouse/financial_statement_extraction/ocr_text_cache_test.rb create mode 100644 test/models/warehouse/financial_statement_extraction/prairie_failed_candidate_filter_test.rb create mode 100644 test/models/warehouse/financial_statement_extraction/quebec_form_pipeline_test.rb create mode 100644 test/models/warehouse/financial_statement_extraction/reviewer_test.rb create mode 100644 test/models/warehouse/financial_statement_extraction/saskatchewan_form_pipeline_test.rb create mode 100644 test/models/warehouse/financial_statement_extraction/scale_detector_test.rb create mode 100644 test/models/warehouse/financial_statement_extraction/stored_headline_pipeline_test.rb create mode 100644 test/scripts/audit_municipal_financial_extraction_coverage_test.rb create mode 100644 test/scripts/process_municipal_financial_statements_test.rb diff --git a/app/controllers/api/v1/warehouse/municipal_financial_statements_controller.rb b/app/controllers/api/v1/warehouse/municipal_financial_statements_controller.rb new file mode 100644 index 00000000..b1159f26 --- /dev/null +++ b/app/controllers/api/v1/warehouse/municipal_financial_statements_controller.rb @@ -0,0 +1,402 @@ +module Api + module V1 + module Warehouse + class MunicipalFinancialStatementsController < CmsBaseController + PROVINCE_NAMES = { + "ab" => "Alberta", "bc" => "British Columbia", "mb" => "Manitoba", + "nb" => "New Brunswick", "nl" => "Newfoundland and Labrador", + "ns" => "Nova Scotia", "nt" => "Northwest Territories", "nu" => "Nunavut", + "on" => "Ontario", "pe" => "Prince Edward Island", "qc" => "Quebec", + "sk" => "Saskatchewan", "yt" => "Yukon" + }.freeze + PROVINCE_SLUGS = PROVINCE_NAMES.transform_values { |name| name.parameterize }.freeze + LOCAL_GOVERNMENT_LEVELS = %w[municipal regional provincial inuit other].freeze + INDEX_PAGE_SIZE = 5_000 + SankeyItem = Data.define(:flow, :origin_flow, :category, :label, :value, :source_page, :position) + + def index + page = [ params.fetch(:page, 1).to_i, 1 ].max + per_page = params.fetch(:per_page, INDEX_PAGE_SIZE).to_i.clamp(1, INDEX_PAGE_SIZE) + scope = latest_approved_extractions_scope + canonical_ids, municipality_count, statement_count = paginated_municipality_ids( + scope, page:, per_page: + ) + statements = scope.where(institution_canonical_id: canonical_ids) + .preload(:institution_release).to_a + institutions = institutions_by_key(statements) + + data = statements.group_by(&:institution_canonical_id).filter_map do |canonical_id, extractions| + institution = newest_institution(institutions, extractions, canonical_id) + serialize_municipality(institution, extractions) if institution + end + + sorted = data.sort_by { |row| [ row[:province], row[:name].downcase ] } + render json: { + data: sorted, + meta: { + municipality_count:, statement_count:, + page:, per_page:, total_pages: (municipality_count.to_f / per_page).ceil + } + } + end + + def show + province = params[:province].to_s.downcase + province = PROVINCE_SLUGS.key(province) || province + canonical_id = "ca/#{province}/#{params[:municipality].to_s.gsub('--', '/')}" + all_statements = latest_approved_extractions(canonical_id: canonical_id) + statements = if params[:year].present? + all_statements.select { |row| row.fiscal_year_end.year == params[:year].to_i } + else + all_statements + end + return render json: { error: "Not found" }, status: :not_found if statements.empty? + + institutions = institutions_by_key(all_statements) + institution = newest_institution(institutions, all_statements, canonical_id) + return render json: { error: "Not found" }, status: :not_found unless institution + + documents = documents_by_key(statements) + facts = ::Warehouse::FinancialStatementFact + .where(financial_statement_extraction_id: statements.map(&:id)) + .order(:concept) + .group_by(&:financial_statement_extraction_id) + line_items = ::Warehouse::FinancialStatementLineItem + .where(financial_statement_extraction_id: statements.map(&:id)) + .order(:flow, :position) + .group_by(&:financial_statement_extraction_id) + context = context_for(institutions.values) + + render json: serialize_municipality(institution, all_statements).merge( + context: context, + statements: statements.sort_by(&:fiscal_year_end).reverse.map do |extraction| + serialize_statement(extraction, documents:, facts:, line_items:, context:) + end + ) + end + + private + + # A document may be re-extracted or carried into a newer ontology release. + # Publish one reviewed result per municipality and fiscal year, favouring the + # newest release and then the most recently reviewed extraction. + def latest_approved_extractions(canonical_id: nil) + scope = latest_approved_extractions_scope + scope = scope.where(institution_canonical_id: canonical_id) if canonical_id + scope.preload(:institution_release).to_a + end + + def latest_approved_extractions_scope + ::Warehouse::FinancialStatementExtraction.publishable_details + .joins(:institution_release) + .joins(<<~SQL.squish) + INNER JOIN warehouse.institutions municipal_institutions + ON municipal_institutions.institution_release_id = warehouse.financial_statement_extractions.institution_release_id + AND municipal_institutions.canonical_id = warehouse.financial_statement_extractions.institution_canonical_id + AND municipal_institutions.government_level IN ('municipal', 'regional', 'provincial', 'inuit', 'other') + SQL + .select(<<~SQL.squish) + DISTINCT ON ( + warehouse.financial_statement_extractions.institution_canonical_id, + warehouse.financial_statement_extractions.fiscal_year_end + ) warehouse.financial_statement_extractions.* + SQL + .order(Arel.sql(<<~SQL.squish)) + warehouse.financial_statement_extractions.institution_canonical_id, + warehouse.financial_statement_extractions.fiscal_year_end, + warehouse.institution_releases.effective_on DESC, + warehouse.financial_statement_extractions.reviewed_at DESC NULLS LAST, + warehouse.financial_statement_extractions.id DESC + SQL + end + + def paginated_municipality_ids(scope, page:, per_page:) + connection = ::Warehouse::FinancialStatementExtraction.connection + latest_sql = scope.to_sql + statement_count = connection.select_value(<<~SQL.squish).to_i + SELECT COUNT(*) FROM (#{latest_sql}) latest_financial_statements + SQL + municipality_count = connection.select_value(<<~SQL.squish).to_i + SELECT COUNT(DISTINCT institution_canonical_id) + FROM (#{latest_sql}) latest_financial_statements + SQL + offset = (page - 1) * per_page + canonical_ids = connection.select_values(<<~SQL.squish) + WITH latest_financial_statements AS (#{latest_sql}) + SELECT latest_financial_statements.institution_canonical_id + FROM latest_financial_statements + INNER JOIN warehouse.institutions index_institutions + ON index_institutions.institution_release_id = latest_financial_statements.institution_release_id + AND index_institutions.canonical_id = latest_financial_statements.institution_canonical_id + GROUP BY latest_financial_statements.institution_canonical_id + ORDER BY + split_part(latest_financial_statements.institution_canonical_id, '/', 2), + LOWER(MAX(COALESCE(index_institutions.name_en, index_institutions.name_fr, ''))), + latest_financial_statements.institution_canonical_id + LIMIT #{per_page} OFFSET #{offset} + SQL + [ canonical_ids, municipality_count, statement_count ] + end + + def institutions_by_key(extractions) + release_ids = extractions.map(&:institution_release_id).uniq + canonical_ids = extractions.map(&:institution_canonical_id).uniq + + ::Warehouse::Institution.where( + institution_release_id: release_ids, + canonical_id: canonical_ids, + government_level: LOCAL_GOVERNMENT_LEVELS + ).index_by { |institution| [ institution.institution_release_id, institution.canonical_id ] } + end + + def serialize_municipality(institution, extractions) + segments = institution.canonical_id.split("/") + province = segments.fetch(1) + slug = segments.drop(2).join("--") + { + canonical_id: institution.canonical_id, + slug: slug, + province: province, + province_name: PROVINCE_NAMES.fetch(province, province.upcase), + province_slug: PROVINCE_SLUGS.fetch(province, province), + name: display_name(institution.name_en.presence || institution.name_fr, province), + name_fr: institution.name_fr, + legal_form: institution.legal_form, + website_url: institution.website_url, + available_years: extractions.map { |row| row.fiscal_year_end.year }.uniq.sort.reverse, + available_periods: extractions.map do |row| + { year: row.fiscal_year_end.year, fiscal_year_end: row.fiscal_year_end } + end.uniq.sort_by { _1[:fiscal_year_end] }.reverse + } + end + + def display_name(name, province) + return name unless province.in?(%w[ab on]) + + name.to_s + .sub(/\A(?:The\s+)?City\s+of\s+/i, "") + .sub(/,\s*City\s+of\z/i, "") + end + + def newest_institution(institutions, extractions, canonical_id) + newest = extractions.max_by do |row| + [ row.institution_release.effective_on, row.reviewed_at || Time.at(0), row.id ] + end + institutions[[ newest.institution_release_id, canonical_id ]] + end + + def documents_by_key(extractions) + ::Warehouse::InstitutionDocument + .where( + institution_release_id: extractions.map(&:institution_release_id).uniq, + canonical_id: extractions.map(&:document_canonical_id).uniq + ) + .includes(:institution_document_assets) + .index_by { |document| [ document.institution_release_id, document.canonical_id ] } + end + + def serialize_statement(extraction, documents:, facts:, line_items:, context:) + document = documents[[ extraction.institution_release_id, extraction.document_canonical_id ]] + asset = document&.institution_document_assets&.find { |row| row.content_sha256 == extraction.asset_sha256 } + extraction_facts = facts.fetch(extraction.id, []) + extraction_line_items = line_items.fetch(extraction.id, []) + + { + fiscal_year: extraction.fiscal_year_end.year, + fiscal_year_end: extraction.fiscal_year_end, + statement_basis: extraction.statement_basis, + language: extraction.language, + source: { + document_id: extraction.document_canonical_id, + page_url: document&.source_page_url, + download_url: asset&.download_url || document&.download_url + }, + facts: extraction_facts.map do |fact| + { + concept: fact.concept, + value: fact.value.to_f, + raw_label: fact.raw_label, + raw_text: fact.raw_text, + statement: fact.statement, + source_page: fact.source_page, + column_year: fact.column_year, + confidence: fact.extraction_confidence&.to_f + } + end, + line_items: extraction_line_items.map do |item| + { + flow: item.flow, category: item.category, label: item.label, + value: item.value.to_f, raw_text: item.raw_text, scale: item.scale, + source_page: item.source_page, column_year: item.column_year, + position: item.position, confidence: item.extraction_confidence&.to_f + } + end, + verification: serialize_verification(extraction), + per_capita: per_capita(extraction_facts, context), + sankey: sankey(extraction_facts, extraction_line_items, checks: extraction.check_results) + } + end + + def serialize_verification(extraction) + checks = ::Warehouse::FinancialStatementExtraction.verification_checks(extraction.check_results) + counts = checks.map { _1[:status] }.tally + failed = checks.length - counts.fetch("pass", 0) - counts.fetch("skip", 0) + + { + status: extraction.status, + reviewed_at: extraction.reviewed_at&.iso8601, + reviewed_by: extraction.reviewed_by, + review_notes: extraction.review_notes, + summary: { + total: checks.length, + pass: counts.fetch("pass", 0), + skip: counts.fetch("skip", 0), + fail: failed + }, + checks: + } + end + + def context_for(institutions) + links = ::Warehouse::InstitutionGeography + .where(institution_id: institutions.map(&:id), role: %w[governs administers]) + .includes(:institution_geography_snapshot) + snapshots = links.map(&:institution_geography_snapshot).uniq(&:id) + profiles = ::Warehouse::CensusProfile + .where( + census_year: snapshots.map(&:census_year).uniq, + geo_level: "csd", + geo_uid: snapshots.map(&:geo_uid) + ) + .order(retrieved_at: :asc, id: :asc) + .index_by { |row| [ row.census_year, row.geo_uid ] } + population_for = ->(snapshot) do + profiles[[ snapshot.census_year, snapshot.geo_uid ]]&.population || snapshot.population + end + area_for = ->(snapshot) do + profiles[[ snapshot.census_year, snapshot.geo_uid ]]&.area_sq_km || snapshot.area_sq_km + end + population = snapshots.sum { |row| population_for.call(row).to_i } + area = snapshots.sum { |row| (area_for.call(row) || 0).to_d } + { + census_year: snapshots.map(&:census_year).compact.max, + population: population.positive? ? population : nil, + area_sq_km: area.positive? ? area.to_f : nil, + population_density_per_sq_km: population.positive? && area.positive? ? (population / area).to_f : nil, + geographies: snapshots.map do |row| + { uid: row.geo_uid, name: row.name_en.presence || row.name_fr, population: population_for.call(row), + area_sq_km: area_for.call(row)&.to_f } + end + } + end + + def per_capita(facts, context) + population = context[:population] + return nil unless population&.positive? + + facts.to_h do |fact| + [ fact.concept, (fact.value.to_d / population).round(2).to_f ] + end + end + + def sankey(facts, line_items, checks:) + return nil if line_items.empty? + + revenue = facts.find { _1.concept == "total_revenue" }&.value&.to_d + spending = facts.find { _1.concept == "total_expenses" }&.value&.to_d + return nil unless revenue&.positive? && spending&.positive? + return nil unless sankey_reconciles?("revenue", revenue, line_items, checks) + return nil unless sankey_reconciles?("expense", spending, line_items, checks) + + revenue = line_items.select { _1.flow == "revenue" }.sum { _1.value.to_d } + spending = line_items.select { _1.flow == "expense" }.sum { _1.value.to_d } + chart_items = normalized_sankey_items(line_items) + inflows = chart_items.select { _1.flow == "revenue" }.sum { _1.value.to_d } + outflows = chart_items.select { _1.flow == "expense" }.sum { _1.value.to_d } + + { + total: [ inflows, outflows ].max.to_f, + revenue: revenue.to_f, + spending: spending.to_f, + revenue_data: sankey_root("revenue", "Inflows", chart_items), + spending_data: sankey_root("expense", "Outflows", chart_items) + } + end + + def sankey_reconciles?(flow, headline, line_items, checks) + rows = line_items.select { |item| item.flow == flow } + return false if rows.empty? + + difference = rows.sum { _1.value.to_d } - headline + return true if difference.abs <= [ headline.abs * BigDecimal("0.001"), BigDecimal("1") ].max + + Array(checks).any? do |check| + check = check.stringify_keys + check["id"] == "line_sum:#{flow}" && check["status"] == "pass" + end + end + + def normalized_sankey_items(line_items) + line_items.filter_map do |item| + value = item.value.to_d + next if value.zero? + + destination_flow = item.flow + category = item.category + if value.negative? + destination_flow = item.flow == "revenue" ? "expense" : "revenue" + category = item.flow == "revenue" ? "Revenue offsets and losses" : "Expense recoveries" + end + SankeyItem.new( + flow: destination_flow, origin_flow: item.flow, category:, label: item.label, + value: value.abs, source_page: item.source_page, position: item.position + ) + end + end + + def sankey_root(flow, label, line_items) + rows = line_items.select { |item| item.flow == flow && item.value.positive? } + children = rows.group_by { sankey_category(_1) }.map do |category, items| + { + id: "#{flow}-#{category.parameterize}", displayName: category, name: category, amount: 0, + children: items.map do |item| + { + id: [ flow, item.origin_flow, category.parameterize, item.label.parameterize, + item.position ].join("-"), + displayName: item.label, name: item.label, + amount: item.value.to_f, + source_page: item.source_page + } + end + } + end + { id: "#{flow}-root", displayName: label, name: label, amount: 0, children: } + end + + def sankey_category(item) + return item.category if item.category.present? && item.category != item.label + + label = item.label + if item.flow == "revenue" + case label + when /tax|taxe|compensation tenant lieu de taxes/i then "Taxes" + when /transfer|grant|subvention|transfert|partage|quote-part/i then "Transfers and grants" + when /service|sale|user fee|tarif|droit|license|licence|permit/i then "Services and fees" + else "Investment and other revenue" + end + else + case label + when /administration|general government|governance/i then "General government" + when /police|fire|sécurité|security|protection/i then "Public safety" + when /transport|road|transit|voirie|circulation/i then "Transportation" + when /water|sewer|waste|environment|hygiène|eau|égout|environnement/i then "Environmental services" + when /health|social|housing|santé|logement/i then "Health, social, and housing" + when /recreation|culture|library|loisir|bibliothèque/i then "Recreation and culture" + else "Other municipal services" + end + end + end + end + end + end +end diff --git a/app/jobs/warehouse/extract_municipal_financial_statements_job.rb b/app/jobs/warehouse/extract_municipal_financial_statements_job.rb new file mode 100644 index 00000000..2bcd5f0d --- /dev/null +++ b/app/jobs/warehouse/extract_municipal_financial_statements_job.rb @@ -0,0 +1,90 @@ +class Warehouse::ExtractMunicipalFinancialStatementsJob < ApplicationJob + include ActiveJob::Continuable + + queue_as :default + self.resume_errors_after_advancing = false + + CIRCUIT_WINDOW = 20 + MAX_FAILURE_RATE = 0.8 + MAX_CONSECUTIVE_MISSING_ASSETS = 3 + FAILURE_STATUSES = %w[failed missing_asset].freeze + + class CircuitOpen < StandardError; end + + def perform(release_version, province:, years: nil, institution_ids: nil, + limit: nil, rerun: "missing", asset_root: nil) + @release = Warehouse::InstitutionRelease.find_by!(version: release_version) + @candidate_set = candidate_set( + province:, years:, institution_ids:, asset_root: + ) + unless @candidate_set.asset_root.directory? + raise CircuitOpen, "asset root is unavailable: #{@candidate_set.asset_root}" + end + @processor = processor(rerun:) + @limit = Integer(limit) if limit + @attempted = 0 + @recent_outcomes = [] + @consecutive_missing_assets = 0 + + step :extract_statements do |step| + @candidate_set.each(start: step.cursor) do |candidate| + result = @processor.call(candidate) + log_result(candidate, result) + track_result!(result) + @attempted += 1 unless result.status == "skipped" + step.advance! from: candidate.document_id + check_circuit! + break if @limit && @attempted >= @limit + end + end + end + + private + + def candidate_set(province:, years:, institution_ids:, asset_root:) + options = { + release: @release, provinces: [ province ], years:, institution_ids: + } + options[:asset_root] = asset_root if asset_root.present? + Warehouse::FinancialStatementExtraction::CandidateSet.new(**options) + end + + def processor(rerun:) + Warehouse::FinancialStatementExtraction::Processor.new(release: @release, rerun:) + end + + def track_result!(result) + @recent_outcomes << result.status + @recent_outcomes.shift while @recent_outcomes.length > CIRCUIT_WINDOW + @consecutive_missing_assets = if result.status == "missing_asset" + @consecutive_missing_assets + 1 + else + 0 + end + end + + def check_circuit! + if @consecutive_missing_assets >= MAX_CONSECUTIVE_MISSING_ASSETS + raise CircuitOpen, "#{@consecutive_missing_assets} consecutive archived assets are missing" + end + return unless @recent_outcomes.length == CIRCUIT_WINDOW + + failures = @recent_outcomes.count { _1.in?(FAILURE_STATUSES) } + if failures.fdiv(CIRCUIT_WINDOW) >= MAX_FAILURE_RATE + raise CircuitOpen, "#{failures} of the last #{CIRCUIT_WINDOW} statements failed" + end + end + + def log_result(candidate, result) + payload = { + event: "municipal_financial_statement_extraction", + document_id: candidate.document_canonical_id, + fiscal_year: candidate.fiscal_year_end.year, + status: result.status, + stage: result.stage, + extraction_id: result.extraction_id, + error: result.error + }.compact + Rails.logger.info(payload.to_json) + end +end diff --git a/app/models/warehouse/census_profile.rb b/app/models/warehouse/census_profile.rb new file mode 100644 index 00000000..0d331934 --- /dev/null +++ b/app/models/warehouse/census_profile.rb @@ -0,0 +1,12 @@ +class Warehouse::CensusProfile < Warehouse::Record + GEO_LEVELS = %w[csd cd pr da].freeze + + validates :census_year, numericality: { only_integer: true, greater_than: 0 } + validates :geo_level, inclusion: { in: GEO_LEVELS } + validates :geo_uid, :source_url, :retrieved_at, presence: true + validates :population, numericality: { only_integer: true, greater_than: 0 } + validates :area_sq_km, numericality: { greater_than: 0 }, allow_nil: true + validates :population_density_per_sq_km, numericality: { greater_than_or_equal_to: 0 }, allow_nil: true + validates :source_sha256, format: { with: /\A[0-9a-f]{64}\z/ }, + uniqueness: { scope: [ :census_year, :geo_level, :geo_uid ] } +end diff --git a/app/models/warehouse/census_profile_importer.rb b/app/models/warehouse/census_profile_importer.rb new file mode 100644 index 00000000..6610b7e8 --- /dev/null +++ b/app/models/warehouse/census_profile_importer.rb @@ -0,0 +1,92 @@ +require "csv" +require "digest" +require "open3" + +class Warehouse::CensusProfileImporter + SOURCE_URL = "https://www12.statcan.gc.ca/census-recensement/2021/dp-pd/prof/details/download-telecharger/comp/getFile.cfm?LANG=E&GEONO=005&FILETYPE=CSV" + CSV_ENTRY = "98-401-X2021005_English_CSV_data.csv" + + class ImportError < StandardError; end + + def initialize(zip_path:, expected_sha256:, retrieved_at:) + @zip_path = Pathname(zip_path).expand_path + @expected_sha256 = expected_sha256 + @retrieved_at = retrieved_at.to_time + end + + def import! + validate_source! + profiles = read_profiles + now = Time.current + rows = profiles.map do |uid, values| + { + census_year: 2021, geo_level: "csd", geo_uid: uid, + population: values.fetch(:population), area_sq_km: values[:area_sq_km], + population_density_per_sq_km: values[:population_density_per_sq_km], + source_url: SOURCE_URL, source_sha256: @expected_sha256, + retrieved_at: @retrieved_at, created_at: now, updated_at: now + } + end + Warehouse::CensusProfile.insert_all( + rows, unique_by: "index_census_profiles_vintage_geography_source" + ) + { + source_url: SOURCE_URL, + source_sha256: @expected_sha256, + census_profile_rows: profiles.length, + stored_profiles: Warehouse::CensusProfile.where( + census_year: 2021, geo_level: "csd", source_sha256: @expected_sha256 + ).count + } + end + + private + + def validate_source! + raise ImportError, "missing Census Profile archive #{@zip_path}" unless @zip_path.file? + actual = Digest::SHA256.file(@zip_path).hexdigest + raise ImportError, "Census Profile SHA-256 mismatch: expected #{@expected_sha256}, got #{actual}" unless actual == @expected_sha256 + end + + def read_profiles + rows = Hash.new { |hash, uid| hash[uid] = {} } + Open3.popen3("unzip", "-p", @zip_path.to_s, CSV_ENTRY) do |stdin, stdout, stderr, wait| + stdin.close + stdout.set_encoding(Encoding::Windows_1252, Encoding::UTF_8, invalid: :replace, undef: :replace) + headers = CSV.parse_line(stdout.gets) + indexes = %w[DGUID GEO_LEVEL CHARACTERISTIC_ID C1_COUNT_TOTAL] + .to_h { |name| [ name, headers.index(name) || raise(ImportError, "missing Census Profile column #{name}") ] } + stdout.each_line do |line| + next unless line.include?("Census subdivision") + next unless line.match?(/,(?:1|6|7),"(?:Population, 2021|Population density per square kilometre|Land area in square kilometres)",/) + + values = CSV.parse_line(line) + next unless values[indexes.fetch("GEO_LEVEL")] == "Census subdivision" + + uid = values[indexes.fetch("DGUID")].to_s.delete_prefix("2021A0005") + next unless uid.length == 7 + + value = values[indexes.fetch("C1_COUNT_TOTAL")] + case values[indexes.fetch("CHARACTERISTIC_ID")] + when "1" + population = Integer(value, exception: false) + rows[uid][:population] = population if population&.positive? + when "6" + density = BigDecimal(value, exception: false) + rows[uid][:population_density_per_sq_km] = density if density && density >= 0 + when "7" + area = BigDecimal(value, exception: false) + rows[uid][:area_sq_km] = area if area&.positive? + end + end + error = stderr.read + raise ImportError, "unzip failed: #{error}" unless wait.value.success? + end + rows.select! { |_uid, values| values[:population]&.positive? } + raise ImportError, "Census Profile contained no CSD profile rows" if rows.empty? + + rows + rescue CSV::MalformedCSVError => error + raise ImportError, "invalid Census Profile CSV: #{error.message}" + end +end diff --git a/app/models/warehouse/financial_statement_extraction.rb b/app/models/warehouse/financial_statement_extraction.rb index 4fe88e5d..c5fcba93 100644 --- a/app/models/warehouse/financial_statement_extraction.rb +++ b/app/models/warehouse/financial_statement_extraction.rb @@ -10,24 +10,44 @@ class Warehouse::FinancialStatementExtraction < Warehouse::Record dependent: :restrict_with_error, inverse_of: :financial_statement_extraction + has_many :financial_statement_line_items, + -> { order(:flow, :position) }, + class_name: "Warehouse::FinancialStatementLineItem", + dependent: :restrict_with_error, + inverse_of: :financial_statement_extraction + has_object :extractor validates :institution_canonical_id, :document_canonical_id, :asset_sha256, :fiscal_year_end, :extractor_version, :status, presence: true validates :asset_sha256, format: { with: /\A[0-9a-f]{64}\z/ }, - uniqueness: { scope: [ :institution_release_id, :extractor_version ] } + uniqueness: { scope: [ :institution_release_id, :extractor_version, :fiscal_year_end ] } validates :status, inclusion: { in: STATUSES } validates :statement_basis, inclusion: { in: STATEMENT_BASES } validates :language, inclusion: { in: LANGUAGES }, allow_nil: true validate :review_fields_are_paired + validate :completed_extractions_have_check_results + validate :approved_extractions_have_review_provenance validate :source_asset_belongs_to_release + validate :document_year_matches_fiscal_year scope :approved, -> { where(status: "approved") } + scope :publishable_details, -> do + approved.where(extractor_version: Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION) + end scope :for_institution, ->(canonical_id) { where(institution_canonical_id: canonical_id) } + def self.verification_checks(check_results) + Array(check_results).map do |check| + check = check.stringify_keys + { id: check["id"], status: check["status"], detail: check["detail"] } + end + end + def approve!(reviewer:, notes: nil) raise ActiveRecord::RecordInvalid, self if reviewed_at.present? raise ArgumentError, "only extracted or needs-review results can be approved" unless status.in?(%w[extracted needs_review]) + raise ArgumentError, "verification check results must be saved before approval" if check_results.blank? update!(status: "approved", reviewed_by: reviewer, reviewed_at: Time.current, review_notes: notes) end @@ -47,6 +67,18 @@ def review_fields_are_paired errors.add(:reviewed_by, "must be present exactly when reviewed_at is present") end + def completed_extractions_have_check_results + return unless status.in?(%w[extracted needs_review approved rejected failed]) && check_results.blank? + + errors.add(:check_results, "must be saved for a completed extraction") + end + + def approved_extractions_have_review_provenance + return unless status == "approved" && (reviewed_at.blank? || reviewed_by.blank?) + + errors.add(:reviewed_at, "and reviewer must be saved before approval") + end + def source_asset_belongs_to_release return unless institution_release && document_canonical_id.present? && asset_sha256.present? @@ -58,4 +90,14 @@ def source_asset_belongs_to_release errors.add(:asset_sha256, "must identify an archived asset on the release document") end end + + def document_year_matches_fiscal_year + return if document_canonical_id.blank? || fiscal_year_end.blank? + + match = document_canonical_id.match(%r{/documents/(?:financial-statements|annual-report)/(\d{4})/}) + return unless match + return if match[1].to_i == fiscal_year_end.year + + errors.add(:fiscal_year_end, "year must match the source document canonical ID") + end end diff --git a/app/models/warehouse/financial_statement_extraction/candidate_set.rb b/app/models/warehouse/financial_statement_extraction/candidate_set.rb new file mode 100644 index 00000000..edb4d585 --- /dev/null +++ b/app/models/warehouse/financial_statement_extraction/candidate_set.rb @@ -0,0 +1,133 @@ +require "digest" + +class Warehouse::FinancialStatementExtraction::CandidateSet + DEFAULT_ASSET_ROOT = Pathname("/Volumes/floppy/york_factory/public_institutions/assets") + PROVINCES = %w[ab bc mb nb nl ns nt nu on pe qc sk yt].freeze + + Candidate = Data.define( + :document_id, :institution_canonical_id, :institution_name, + :document_canonical_id, :asset_sha256, :fiscal_year_end, + :pdf_path, :population + ) + + attr_reader :release, :provinces, :years, :institution_ids, :asset_root + + def initialize(release:, provinces: nil, years: nil, institution_ids: nil, + asset_root: ENV.fetch("PUBLIC_INSTITUTION_ASSET_ROOT", DEFAULT_ASSET_ROOT.to_s)) + @release = release + @provinces = Array(provinces).presence&.map { normalize_province(_1) }&.uniq + @years = Array(years).presence&.map { Integer(_1) }&.uniq + @institution_ids = Array(institution_ids).presence&.map(&:to_s)&.uniq + @asset_root = Pathname(asset_root).expand_path + @population_by_institution_id = {} + end + + def count + return relation.count unless years + + each.count + end + + def each(start: nil) + return enum_for(__method__, start:) unless block_given? + + scope = relation.order(:id) + scope = scope.where("warehouse.institution_documents.id >= ?", start) if start + scope.find_each do |document| + fiscal_year_end = fiscal_year_end_for(document) + next if years && !fiscal_year_end.year.in?(years) + + asset = document.institution_document_assets.find(&:preferred?) + yield Candidate.new( + document_id: document.id, + institution_canonical_id: document.institution.canonical_id, + institution_name: document.institution.name_en.presence || document.institution.name_fr, + document_canonical_id: document.canonical_id, + asset_sha256: asset.content_sha256, + fiscal_year_end:, + pdf_path: safe_asset_path(asset.archive_path), + population: population_for(document.institution) + ) + end + end + + def audit(verify_hashes: false) + result = { candidates: 0, missing_files: [], size_mismatches: [], hash_mismatches: [] } + each do |candidate| + result[:candidates] += 1 + unless candidate.pdf_path.file? + result[:missing_files] << candidate.document_canonical_id + next + end + + asset = release.institution_document_assets.find_by!( + institution_document_id: candidate.document_id, preferred: true + ) + if candidate.pdf_path.size != asset.byte_size + result[:size_mismatches] << candidate.document_canonical_id + end + if verify_hashes && Digest::SHA256.file(candidate.pdf_path).hexdigest != candidate.asset_sha256 + result[:hash_mismatches] << candidate.document_canonical_id + end + end + result + end + + private + + def relation + scope = release.institution_documents + .joins(:institution, :institution_document_assets) + .includes(:institution, :institution_document_assets) + .where(document_type: "financial-statements") + .where("warehouse.institution_document_assets" => { preferred: true, mime_type: "application/pdf" }) + if provinces + patterns = provinces.map { "ca/#{_1}/%" } + clauses = patterns.map { "warehouse.institutions.canonical_id LIKE ?" }.join(" OR ") + scope = scope.where(clauses, *patterns) + end + scope = scope.where("warehouse.institutions" => { canonical_id: institution_ids }) if institution_ids + scope + end + + def fiscal_year_end_for(document) + return document.fiscal_period_end if document.fiscal_period_end + + match = document.canonical_id.match(%r{/documents/financial-statements/(\d{4})/}) + raise ArgumentError, "document has no fiscal year: #{document.canonical_id}" unless match + + Date.new(Integer(match[1]), 12, 31) + end + + def safe_asset_path(relative_path) + path = asset_root.join(relative_path).expand_path + unless path.to_s.start_with?("#{asset_root}/") + raise ArgumentError, "asset path escapes root: #{relative_path}" + end + + path + end + + def population_for(institution) + @population_by_institution_id.fetch(institution.id) do + snapshots = institution.institution_geographies + .where(role: %w[governs administers]) + .includes(:institution_geography_snapshot) + .map(&:institution_geography_snapshot) + profiles = Warehouse::CensusProfile.where( + census_year: snapshots.map(&:census_year), geo_level: "csd", geo_uid: snapshots.map(&:geo_uid) + ).index_by { [ _1.census_year, _1.geo_uid ] } + population = snapshots.sum do |snapshot| + profiles[[ snapshot.census_year, snapshot.geo_uid ]]&.population.to_i.nonzero? || snapshot.population.to_i + end + @population_by_institution_id[institution.id] = population.positive? ? population : nil + end + end + + def normalize_province(value) + province = value.to_s.downcase + raise ArgumentError, "unsupported province #{value.inspect}" unless province.in?(PROVINCES) + + province + end +end diff --git a/app/models/warehouse/financial_statement_extraction/candidate_window.rb b/app/models/warehouse/financial_statement_extraction/candidate_window.rb new file mode 100644 index 00000000..54fdee86 --- /dev/null +++ b/app/models/warehouse/financial_statement_extraction/candidate_window.rb @@ -0,0 +1,28 @@ +require "set" + +class Warehouse::FinancialStatementExtraction::CandidateWindow + attr_reader :start, :stop_before, :document_ids, :excluded_document_ids + + def initialize(start: nil, stop_before: nil, document_ids: nil, excluded_document_ids: nil) + @start = start + @stop_before = stop_before + @document_ids = Array(document_ids).map { Integer(_1) }.to_set + @excluded_document_ids = Array(excluded_document_ids).map { Integer(_1) }.to_set + end + + def before_start?(candidate) + start.present? && candidate.document_id < start + end + + def at_or_after_stop?(candidate) + stop_before.present? && candidate.document_id >= stop_before + end + + def selected?(candidate) + document_ids.empty? || document_ids.include?(candidate.document_id) + end + + def excluded?(candidate) + excluded_document_ids.include?(candidate.document_id) + end +end diff --git a/app/models/warehouse/financial_statement_extraction/coverage_audit.rb b/app/models/warehouse/financial_statement_extraction/coverage_audit.rb new file mode 100644 index 00000000..d0466ace --- /dev/null +++ b/app/models/warehouse/financial_statement_extraction/coverage_audit.rb @@ -0,0 +1,170 @@ +class Warehouse::FinancialStatementExtraction::CoverageAudit + TERMINAL_STATUSES = %w[extracted needs_review approved rejected failed].freeze + + attr_reader :release, :provinces + + def initialize(release:, provinces: nil) + @release = release + @provinces = provinces + end + + def payload + candidate_rows = candidates.each.to_a + candidate_groups = candidate_rows.group_by { identity_key(_1) } + predicted_owner_by_key = candidate_groups.transform_values { _1.min_by(&:document_id) } + detailed_by_key = extraction_scope( + Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION + ).includes(:financial_statement_facts, :financial_statement_line_items).index_by do |extraction| + identity_key(extraction) + end + headline_by_key = extraction_scope( + Warehouse::FinancialStatementExtraction::Pipeline::EXTRACTOR_VERSION + ).index_by { identity_key(_1) } + records = candidate_rows.map do |candidate| + record_for( + candidate:, candidate_groups:, predicted_owner_by_key:, detailed_by_key:, headline_by_key: + ) + end + + { + generated_at: Time.current.iso8601, + release: release.version, + extractor_version: Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION, + definitions: { + preferred_asset: "one preferred archived PDF candidate, including document variants", + institution_year: "one reporting institution and fiscal year, matching the API publication surface", + approved: "publishable only after saved verification checks and reviewer provenance", + shared_asset: "a non-owner candidate sharing extraction identity with another preferred PDF; " \ + "the persisted row wins, otherwise the lowest document id predicts ownership" + }, + totals: summarize(records), + provinces: records.group_by { _1.fetch(:province) }.sort.to_h do |province, rows| + [ province, summarize(rows) ] + end, + records: + } + end + + private + + def candidates + Warehouse::FinancialStatementExtraction::CandidateSet.new(release:, provinces:) + end + + def extraction_scope(extractor_version) + release.financial_statement_extractions.where(extractor_version:) + end + + def identity_key(candidate_or_extraction) + [ candidate_or_extraction.asset_sha256, candidate_or_extraction.fiscal_year_end ] + end + + def record_for(candidate:, candidate_groups:, predicted_owner_by_key:, detailed_by_key:, headline_by_key:) + key = identity_key(candidate) + detailed = detailed_by_key[key] + headline = headline_by_key[key] + extraction = detailed || headline + predicted_owner = predicted_owner_by_key.fetch(key) + owner_document = extraction&.document_canonical_id || predicted_owner.document_canonical_id + shared_asset = candidate.document_canonical_id != owner_document + shared_owner = candidate_groups.fetch(key).find do |group_candidate| + group_candidate.document_canonical_id == owner_document + end || predicted_owner + status = record_status(shared_asset:, detailed:, headline:) + checks = Warehouse::FinancialStatementExtraction.verification_checks(extraction&.check_results) + check_statuses = checks.map { _1.fetch(:status) }.tally + shared_with_institution = if shared_asset + extraction&.institution_canonical_id || shared_owner.institution_canonical_id + end + + { + province: candidate.institution_canonical_id.split("/").fetch(1), + institution_canonical_id: candidate.institution_canonical_id, + document_canonical_id: candidate.document_canonical_id, + asset_sha256: candidate.asset_sha256, + fiscal_year: candidate.fiscal_year_end.year, + extraction_id: extraction&.id, + extraction_stage: extraction_stage(shared_asset:, detailed:, headline:), + status:, + shared_extraction_status: shared_asset ? extraction&.status : nil, + shared_with_document_canonical_id: shared_asset ? owner_document : nil, + shared_with_institution_canonical_id: shared_with_institution, + parser: extraction&.llm_response_snapshot&.fetch("parser", nil), + error: extraction&.error_message, + reviewed_by: extraction&.reviewed_by, + reviewed_at: extraction&.reviewed_at&.iso8601, + verification: { + total: checks.length, + pass: check_statuses.fetch("pass", 0), + skip: check_statuses.fetch("skip", 0), + fail: checks.length - check_statuses.fetch("pass", 0) - check_statuses.fetch("skip", 0), + checks: + }, + fact_count: shared_asset ? 0 : (detailed&.financial_statement_facts&.length || 0), + line_item_count: shared_asset ? 0 : (detailed&.financial_statement_line_items&.length || 0) + } + end + + def record_status(shared_asset:, detailed:, headline:) + return "shared_asset" if shared_asset + return detailed.status if detailed + return "failed_headline_gate" if headline&.status == "failed" + return "headline_#{headline.status}" if headline + + "unattempted" + end + + def extraction_stage(shared_asset:, detailed:, headline:) + return "shared_asset" if shared_asset + return "detailed" if detailed + + "headline_gate" if headline + end + + def summarize(rows) + institution_years = rows.group_by do |row| + [ row.fetch(:institution_canonical_id), row.fetch(:fiscal_year) ] + end + status_counts = rows.map { _1.fetch(:status) }.tally + published_institution_year_count = institution_years.count do |_, variants| + variants.any? { _1.fetch(:status) == "approved" } + end + approved_asset_count = status_counts.fetch("approved", 0) + unattempted_asset_count = status_counts.fetch("unattempted", 0) + + { + preferred_asset_count: rows.length, + institution_year_count: institution_years.length, + published_institution_year_count:, + status_counts:, + parser_counts: rows.filter_map { _1.fetch(:parser) }.tally, + reconciliation: { + classified_asset_count: rows.length - unattempted_asset_count, + unclassified_asset_count: unattempted_asset_count, + approved_asset_count:, + approved_duplicate_variant_count: approved_asset_count - published_institution_year_count, + awaiting_review_asset_count: status_counts.fetch("extracted", 0) + + status_counts.fetch("needs_review", 0), + rejected_asset_count: status_counts.fetch("rejected", 0) + }, + approved_without_checks: without_checks(rows, "approved"), + failed_headline_gate_without_checks: without_checks(rows, "failed_headline_gate"), + shared_asset_with_terminal_extraction_without_checks: rows.count do |row| + row.fetch(:status) == "shared_asset" && + row.fetch(:shared_extraction_status).in?(TERMINAL_STATUSES) && + row.dig(:verification, :total).zero? + end, + approved_without_deterministic_reviewer: rows.count do |row| + row.fetch(:status) == "approved" && !row.fetch(:reviewed_by).in?( + Warehouse::FinancialStatementExtraction::Reviewer::DETERMINISTIC_REVIEWERS + ) + end + } + end + + def without_checks(rows, status) + rows.count do |row| + row.fetch(:status) == status && row.dig(:verification, :total).zero? + end + end +end diff --git a/app/models/warehouse/financial_statement_extraction/detailed_pipeline.rb b/app/models/warehouse/financial_statement_extraction/detailed_pipeline.rb new file mode 100644 index 00000000..a96cd38f --- /dev/null +++ b/app/models/warehouse/financial_statement_extraction/detailed_pipeline.rb @@ -0,0 +1,312 @@ +require "timeout" +require "fileutils" +require "tmpdir" + +class Warehouse::FinancialStatementExtraction::DetailedPipeline + EXTRACTOR_VERSION = "detailed-psas-v1" + DEFAULT_MODEL = Warehouse::FinancialStatementExtraction::Pipeline::DEFAULT_MODEL + MODEL_TIMEOUT = ENV.fetch("MUNICIPAL_FINANCIAL_MODEL_TIMEOUT", 300).to_i + MAX_OUTPUT_TOKENS = ENV.fetch("MUNICIPAL_FINANCIAL_DETAIL_MAX_OUTPUT_TOKENS", 32_768).to_i + MAX_DETAIL_PAGES_PER_FLOW = 4 + MAX_LINE_ITEMS_PER_FLOW = 100 + FLOW_CONCURRENCIES = [ 1, 2 ].freeze + Result = Data.define( + :status, :facts, :line_items, :checks, :prompt, :response, + :locator_result, :language, :statement_basis + ) + FlowJob = Data.define(:flow, :pages, :prompt) + PreparedFlowJob = Data.define(:flow, :pages, :prompt, :pdf_path) + + class ResponseError < StandardError; end + + def initialize(pdf_path:, institution_canonical_id:, institution_name:, document_canonical_id:, + asset_sha256:, fiscal_year_end:, population: nil, model: DEFAULT_MODEL, + headline_pipeline: nil, llm_client: nil, llm_client_factory: nil, page_locator: nil, + flow_concurrency: ENV.fetch("MUNICIPAL_FINANCIAL_DETAIL_FLOW_CONCURRENCY", 1), flow_reporter: nil) + @pdf_path = Pathname(pdf_path) + @institution_canonical_id = institution_canonical_id + @institution_name = institution_name + @document_canonical_id = document_canonical_id + @asset_sha256 = asset_sha256 + @fiscal_year_end = fiscal_year_end.to_date + @population = population + @model = model + @page_locator = page_locator || Warehouse::FinancialStatementExtraction::PageLocator.new(@pdf_path) + @headline_pipeline = headline_pipeline || Warehouse::FinancialStatementExtraction::Pipeline.new( + pdf_path:, institution_canonical_id:, institution_name:, document_canonical_id:, + asset_sha256:, fiscal_year_end:, population:, model:, page_locator: @page_locator + ) + @flow_concurrency = normalize_flow_concurrency(flow_concurrency) + if llm_client && llm_client_factory + raise ArgumentError, "provide llm_client or llm_client_factory, not both" + end + if @flow_concurrency == 2 && llm_client + raise ArgumentError, "concurrent flow extraction requires llm_client_factory" + end + @llm_client = llm_client || method(:call_ruby_llm) + @llm_client_factory = llm_client_factory || method(:new_ruby_llm_client) + @flow_reporter = flow_reporter || ->(event) { warn(event.to_json) } + @flow_reporter_mutex = Mutex.new + end + + def run + headline = @headline_pipeline.run + locator = headline.locator_result + jobs = Warehouse::FinancialStatementLineItem::FLOWS.map do |flow| + pages = select_detail_pages(locator, flow).freeze + FlowJob.new(flow:, pages:, prompt: build_prompt(pages, locator.page_texts, flow).freeze) + end.freeze + flow_responses = extract_flows(jobs) + prompts = {} + responses = {} + line_items = [] + jobs.each do |job| + response = flow_responses.fetch(job.flow) + validate_response!(response, job.pages, job.flow) + prompts[job.flow] = job.prompt + responses[job.flow] = response + line_items.concat(normalize_line_items( + response.fetch("line_items"), job.pages, locator.page_texts + )) + end + flags = headline.response.slice( + "remeasurement_present", "operations_adjustment_present", "rollforward_adjustment_present" + ).symbolize_keys + flags[:single_component_concepts] = + Warehouse::FinancialStatementExtraction::Pipeline.single_component_concepts(headline.response) + validator = Warehouse::FinancialStatementExtraction::Validator.new( + facts: headline.facts, line_items:, fiscal_year: @fiscal_year_end.year, + population: @population, page_texts: locator.page_texts, flags: + ) + checks = [ headline.checks.find { |check| check[:id] == "source_identity" }, *validator.validate ].compact + status = validator.acceptable?(checks) ? "extracted" : "needs_review" + status = "needs_review" if headline.status == "needs_review" + Result.new( + status:, facts: headline.facts, line_items:, checks:, + prompt: prompts.map { |flow, value| "REQUESTED FLOW: #{flow}\n#{value}" }.join("\n\n"), + response: { "headline" => headline.response, "details" => responses }, + locator_result: locator, language: headline.language, statement_basis: headline.statement_basis + ) + rescue JSON::ParserError, KeyError, ArgumentError, ResponseError, Timeout::Error => error + raise ResponseError, error.message + end + + private + + def normalize_flow_concurrency(value) + concurrency = Integer(value) + return concurrency if concurrency.in?(FLOW_CONCURRENCIES) + + raise ArgumentError, "flow_concurrency must be 1 or 2" + rescue TypeError, ArgumentError + raise ArgumentError, "flow_concurrency must be 1 or 2" + end + + def extract_flows(jobs) + return jobs.to_h { |job| [ job.flow, extract_serial_flow(job) ] } if @flow_concurrency == 1 + + Dir.mktmpdir("financial-statement-detail-flows") do |directory| + prepared = jobs.map do |job| + destination = Pathname(directory).join("#{job.flow}.pdf") + @page_locator.with_excerpt(job.pages) { |excerpt| FileUtils.cp(excerpt, destination) } + PreparedFlowJob.new( + flow: job.flow, pages: job.pages, prompt: job.prompt, pdf_path: destination.freeze + ) + end.freeze + clients = prepared.map { @llm_client_factory.call } + unless clients.all? { _1.respond_to?(:call) } && clients.map(&:object_id).uniq.length == clients.length + raise ArgumentError, "llm_client_factory must return a distinct callable per flow" + end + + threads = prepared.zip(clients).map do |job, client| + Thread.new { extract_flow_response(job, client) }.tap { _1.report_on_exception = false } + end + threads.each do |thread| + thread.join + rescue StandardError + # Join every worker so its request and temporary excerpt are finished + # before the first error is re-raised below. + end + prepared.map.with_index { |job, index| [ job.flow, threads.fetch(index).value ] }.to_h + end + end + + def extract_serial_flow(job) + @page_locator.with_excerpt(job.pages) do |excerpt| + extract_flow_response( + PreparedFlowJob.new(flow: job.flow, pages: job.pages, prompt: job.prompt, pdf_path: excerpt), + @llm_client + ) + end + end + + def extract_flow_response(job, client) + started = Process.clock_gettime(Process::CLOCK_MONOTONIC) + raw = Timeout.timeout(MODEL_TIMEOUT) { client.call(prompt: job.prompt, pdf_path: job.pdf_path.to_s) } + response = raw.respond_to?(:content) ? raw.content : raw + response = JSON.parse(response) if response.is_a?(String) + report_flow(job.flow, "success", started) + response + rescue => error + report_flow(job.flow, "failure", started, error: error) + raise + end + + def report_flow(flow, outcome, started, error: nil) + elapsed = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1_000).round + event = { + financial_statement_detail_flow: outcome, flow:, elapsed_ms: elapsed, + error_class: error&.class&.name + }.compact + @flow_reporter_mutex.synchronize { @flow_reporter.call(event) } + rescue StandardError + nil + end + + # Each invocation constructs a new RubyLLM chat, so concurrent flow requests + # cannot share message history or adapter state. + def new_ruby_llm_client + method(:call_ruby_llm) + end + + def select_detail_pages(locator, flow) + operations_text = locator.page_texts.fetch(locator.operations_page) + return [ locator.operations_page ] if primary_has_detail_rows?(operations_text, flow) + + flow_pattern = flow == "revenue" ? /revenues?|revenus|produits/i : /expenses?|expenditures?|dépenses|depenses|charges/i + schedule_pattern = /schedule|appendix|annexe|cédule|cedule/i + supporting = locator.page_texts.filter_map do |page, text| + next if page == locator.operations_page + + heading = text.lines.first(20).join + next unless heading.match?(schedule_pattern) && text.match?(flow_pattern) + next unless text.match?(/(?= 3 } + [ page, numeric_cells ] + end + selected = supporting.sort_by { |page, score| [ -score, page ] } + .first(MAX_DETAIL_PAGES_PER_FLOW - 1).map(&:first) + [ locator.operations_page, *selected ].uniq.sort + end + + def primary_has_detail_rows?(text, flow) + start_pattern = if flow == "revenue" + /\A\s*(?:revenues?|revenus|produits)\s*\z/i + else + /\A\s*(?:expenses?|expenditures?|d[eé]penses|charges)\s*\z/i + end + finish_pattern = if flow == "revenue" + /\A\s*(?:expenses?|expenditures?|d[eé]penses|charges)\s*\z/i + else + /\A\s*(?:total\s+)?(?:expenses?|expenditures?|d[eé]penses|charges)\b|\A\s*(?:annual|excess|exc[eé]dent)/i + end + lines = text.lines + start_index = lines.index { _1.match?(start_pattern) } + return false unless start_index + + finish_index = lines.each_index.find { _1 > start_index && lines[_1].match?(finish_pattern) } || lines.length + rows = lines[(start_index + 1)...finish_index].count do |line| + line.match?(/[[:alpha:]]/) && line.scan(/\(?-?\d[\d,.]*\)?/).count { _1.scan(/\d/).length >= 3 } >= 2 + end + rows >= 2 + end + + def call_ruby_llm(prompt:, pdf_path:) + RubyLLM.chat(model: @model) + .with_temperature(0) + .with_thinking(effort: :low) + .with_params(generationConfig: { maxOutputTokens: MAX_OUTPUT_TOKENS }) + .with_schema(Warehouse::FinancialStatementExtraction::DetailedResponseSchema) + .ask(prompt, with: pdf_path) + end + + def build_prompt(pages, page_texts, flow) + page_map = pages.map.with_index(1) { |physical, excerpt| "excerpt page #{excerpt} = archived PDF page #{physical}" }.join("\n") + source_text = pages.map.with_index(1) do |physical, excerpt| + "EXCERPT PAGE #{excerpt} (ARCHIVED PDF PAGE #{physical})\n#{page_texts.fetch(physical)}" + end.join("\n\n") + <<~PROMPT + Extract the detailed current-year #{flow} leaf line items from this Canadian municipal financial statement. + + Institution: #{@institution_name} (#{@institution_canonical_id}) + Fiscal year: #{@fiscal_year_end.year} + + PAGE MAP + #{page_map} + + REQUESTED FLOW: #{flow} + + Rules: + - Use only the #{@fiscal_year_end.year} actual column, never budget or comparative values. + - Extract printed leaf line items that add to the consolidated total revenue or total expenses. + - Do not return totals or subtotals as line items. Preserve negative recoveries or adjustments. + - Return only #{flow} rows and set flow to #{flow}. category is the nearest printed group, function, segment, or schedule heading. + - If no group is printed, repeat the label as category. Never invent a program or category. + - label is the exact printed row label. raw_text is only the exact printed numeric cell for #{@fiscal_year_end.year} + (for example "$ 2,332,958" or "(45,957)"); never put a row label in raw_text. + - Copy label punctuation literally, including ampersands, apostrophes, hyphens, and note parentheses; do not expand, + contract, correct, or paraphrase the printed label. + - scale is 1, 1000, or 1000000 from the page heading. + - excerpt_page uses the page map above. column_year preserves the printed column heading. + When the heading is stacked across lines, combine it (for example "2023 Actual"), so column_year always includes #{@fiscal_year_end.year}. + - The first excerpt is the primary statement of operations. If it prints leaf rows for this flow, use only those + primary-statement rows and ignore supporting schedules. Otherwise use exactly one complete supporting schedule. + - On a primary statement with printed totals, revenue rows must be strictly between the Revenue heading and Total Revenue, + and expense rows must be strictly between the Expenses heading and Total Expenses or Total Expenditures. Exclude every + row after those totals, including later Other revenue (expenditure), capital transfers, and disposal gains or losses. + If the primary statement has no printed flow total, stop at the next flow heading or annual surplus line instead. + - Never combine duplicate presentations of the same values or mix a primary statement with its supporting schedule. + - Omit ambiguous values instead of calculating or guessing. + - Recognize French revenus, produits, charges, and dépenses. + + SOURCE TEXT + #{source_text} + PROMPT + end + + def validate_response!(response, pages, expected_flow) + raise ResponseError, "response must be an object" unless response.is_a?(Hash) + raise ResponseError, "response fiscal year does not match" unless Integer(response.fetch("fiscal_year")) == @fiscal_year_end.year + items = response.fetch("line_items") + raise ResponseError, "line_items must not be empty" unless items.is_a?(Array) && items.any? + raise ResponseError, "too many line items" if items.length > MAX_LINE_ITEMS_PER_FLOW + identities = items.map { |item| item.values_at("flow", "category", "label") } + raise ResponseError, "duplicate line items" unless identities.uniq.length == identities.length + items.each do |item| + raise ResponseError, "invalid flow" unless item.fetch("flow").in?(Warehouse::FinancialStatementLineItem::FLOWS) + raise ResponseError, "unexpected flow" unless item.fetch("flow") == expected_flow + raise ResponseError, "blank category or label" if item.fetch("category").blank? || item.fetch("label").blank? + raise ResponseError, "invalid scale" unless Integer(item.fetch("scale")).in?(Warehouse::FinancialStatementLineItem::SCALES) + page = Integer(item.fetch("excerpt_page")) + raise ResponseError, "excerpt page out of range" unless page.between?(1, pages.length) + confidence = Float(item.fetch("confidence")) + raise ResponseError, "confidence out of range" unless confidence.between?(0, 1) + end + end + + def normalize_line_items(items, pages, page_texts) + positions = Hash.new(0) + items.filter_map do |item| + next if Warehouse::FinancialStatementExtraction::NumberParser.null_marker?(item.fetch("raw_text")) + + flow = item.fetch("flow") + position = positions[flow] + positions[flow] += 1 + source_page = pages.fetch(Integer(item.fetch("excerpt_page")) - 1) + { + flow:, category: item.fetch("category"), label: item.fetch("label"), + raw_text: item.fetch("raw_text"), + value: Warehouse::FinancialStatementExtraction::NumberParser.parse(item.fetch("raw_text")) * Integer(item.fetch("scale")), + scale: Integer(item.fetch("scale")), + source_page:, + column_year: Warehouse::FinancialStatementExtraction::Pipeline.normalize_column_year( + item.fetch("column_year"), fiscal_year: @fiscal_year_end.year, + page_text: page_texts.fetch(source_page) + ), + position:, + extraction_confidence: BigDecimal(item.fetch("confidence").to_s) + } + end + end +end diff --git a/app/models/warehouse/financial_statement_extraction/detailed_response_schema.rb b/app/models/warehouse/financial_statement_extraction/detailed_response_schema.rb new file mode 100644 index 00000000..26405ac3 --- /dev/null +++ b/app/models/warehouse/financial_statement_extraction/detailed_response_schema.rb @@ -0,0 +1,17 @@ +require "ruby_llm/schema" + +class Warehouse::FinancialStatementExtraction::DetailedResponseSchema < RubyLLM::Schema + integer :fiscal_year + array :line_items, min_items: 1 do + object do + string :flow, enum: Warehouse::FinancialStatementLineItem::FLOWS + string :category + string :label + string :raw_text + integer :scale + integer :excerpt_page, minimum: 1 + string :column_year + number :confidence, minimum: 0, maximum: 1 + end + end +end diff --git a/app/models/warehouse/financial_statement_extraction/extractor.rb b/app/models/warehouse/financial_statement_extraction/extractor.rb index b7e943aa..8e83155c 100644 --- a/app/models/warehouse/financial_statement_extraction/extractor.rb +++ b/app/models/warehouse/financial_statement_extraction/extractor.rb @@ -5,6 +5,31 @@ def extract(pdf_path:, institution_name:, population: nil) raise ArgumentError, "reviewed extractions are immutable; create a new extractor version" if financial_statement_extraction.reviewed_at.present? financial_statement_extraction.update!(status: "extracting", error_message: nil) + result = headline_pipeline( + pdf_path:, + institution_canonical_id: financial_statement_extraction.institution_canonical_id, + institution_name:, + document_canonical_id: financial_statement_extraction.document_canonical_id, + asset_sha256: financial_statement_extraction.asset_sha256, + fiscal_year_end: financial_statement_extraction.fiscal_year_end, + population:, + model: financial_statement_extraction.llm_model.presence || Warehouse::FinancialStatementExtraction::Pipeline::DEFAULT_MODEL + ).run + + persist_headline(result) + result + rescue => error + record_failure(error, stage: "headline_extraction") + raise + end + + + def revalidate_headline(pdf_path:, institution_name:, population: nil) + raise ArgumentError, "reviewed extractions are immutable; create a new extractor version" if financial_statement_extraction.reviewed_at.present? + unless financial_statement_extraction.extractor_version == Warehouse::FinancialStatementExtraction::Pipeline::EXTRACTOR_VERSION + raise ArgumentError, "only headline extractions can be revalidated" + end + result = Warehouse::FinancialStatementExtraction::Pipeline.new( pdf_path:, institution_canonical_id: financial_statement_extraction.institution_canonical_id, @@ -14,11 +39,48 @@ def extract(pdf_path:, institution_name:, population: nil) fiscal_year_end: financial_statement_extraction.fiscal_year_end, population:, model: financial_statement_extraction.llm_model.presence || Warehouse::FinancialStatementExtraction::Pipeline::DEFAULT_MODEL + ).revalidate( + response: financial_statement_extraction.llm_response_snapshot, + prompt: financial_statement_extraction.llm_prompt_snapshot&.fetch("prompt", nil), + source_pages: financial_statement_extraction.financial_statement_facts.to_h do |fact| + [ fact.concept, fact.source_page ] + end + ) + persist_headline(result) + result + end + + def extract_detailed(pdf_path:, institution_name:, population: nil) + raise ArgumentError, "reviewed extractions are immutable; create a new extractor version" if financial_statement_extraction.reviewed_at.present? + + financial_statement_extraction.update!(status: "extracting", error_message: nil) + page_locator = Warehouse::FinancialStatementExtraction::PageLocator.new(pdf_path) + headline = financial_statement_extraction.institution_release.financial_statement_extractions + .where(status: Warehouse::FinancialStatementExtraction::Processor::DETAIL_HEADLINE_STATUSES).find_by( + asset_sha256: financial_statement_extraction.asset_sha256, + extractor_version: Warehouse::FinancialStatementExtraction::Pipeline::EXTRACTOR_VERSION, + fiscal_year_end: financial_statement_extraction.fiscal_year_end + ) + headline_pipeline = headline && Warehouse::FinancialStatementExtraction::StoredHeadlinePipeline.new( + extraction: headline, pdf_path:, page_locator:, allow_needs_review: headline.status == "needs_review" + ) + result = detailed_pipeline( + pdf_path:, + institution_canonical_id: financial_statement_extraction.institution_canonical_id, + institution_name:, + document_canonical_id: financial_statement_extraction.document_canonical_id, + asset_sha256: financial_statement_extraction.asset_sha256, + fiscal_year_end: financial_statement_extraction.fiscal_year_end, + population:, + model: financial_statement_extraction.llm_model.presence || Warehouse::FinancialStatementExtraction::DetailedPipeline::DEFAULT_MODEL, + page_locator:, headline_pipeline: ).run financial_statement_extraction.transaction do - financial_statement_extraction.financial_statement_facts.delete_all + Warehouse::FinancialStatementFact.where(financial_statement_extraction:).delete_all + Warehouse::FinancialStatementLineItem.where(financial_statement_extraction:).delete_all result.facts.each { |attributes| financial_statement_extraction.financial_statement_facts.create!(attributes) } + result.line_items.each { |attributes| financial_statement_extraction.financial_statement_line_items.create!(attributes) } financial_statement_extraction.update!( status: result.status, statement_basis: result.statement_basis, @@ -31,9 +93,64 @@ def extract(pdf_path:, institution_name:, population: nil) end result rescue => error - unless financial_statement_extraction.reviewed_at.present? - financial_statement_extraction.update!(status: "failed", error_message: "#{error.class}: #{error.message}") - end + record_failure(error, stage: "detailed_extraction") raise end + + private + + def record_failure(error, stage:) + return if financial_statement_extraction.reviewed_at.present? + + detail = "#{error.class}: #{error.message}" + financial_statement_extraction.update!( + status: "failed", + check_results: [ { id: stage, status: "fail", detail: } ], + error_message: detail + ) + end + + def persist_headline(result) + financial_statement_extraction.transaction do + Warehouse::FinancialStatementFact.where(financial_statement_extraction:).delete_all + result.facts.each { |attributes| financial_statement_extraction.financial_statement_facts.create!(attributes) } + financial_statement_extraction.update!( + status: result.status, + statement_basis: result.statement_basis, + language: result.language, + check_results: result.checks, + llm_prompt_snapshot: { prompt: result.prompt }, + llm_response_snapshot: result.response, + error_message: nil + ) + end + end + + def headline_pipeline(**attributes) + fallback = Warehouse::FinancialStatementExtraction::Pipeline.new(**attributes) + pipeline_class = deterministic_pipeline_class(attributes) + return fallback unless pipeline_class + + deterministic = pipeline_class.new(**attributes) + Warehouse::FinancialStatementExtraction::FallbackPipeline.new( + primary: pipeline_class::Headline.new(deterministic), fallback:, on: pipeline_class::Unsupported + ) + end + + def detailed_pipeline(**attributes) + fallback = Warehouse::FinancialStatementExtraction::DetailedPipeline.new(**attributes) + pipeline_class = deterministic_pipeline_class(attributes) + return fallback unless pipeline_class + + Warehouse::FinancialStatementExtraction::FallbackPipeline.new( + primary: pipeline_class.new(**attributes), fallback:, on: pipeline_class::Unsupported + ) + end + + def deterministic_pipeline_class(attributes) + [ + Warehouse::FinancialStatementExtraction::QuebecFormPipeline, + Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline + ].find { _1.applicable?(**attributes.slice(:institution_canonical_id, :fiscal_year_end)) } + end end diff --git a/app/models/warehouse/financial_statement_extraction/failed_candidate_filter.rb b/app/models/warehouse/financial_statement_extraction/failed_candidate_filter.rb new file mode 100644 index 00000000..4858d2d8 --- /dev/null +++ b/app/models/warehouse/financial_statement_extraction/failed_candidate_filter.rb @@ -0,0 +1,138 @@ +require "set" + +class Warehouse::FinancialStatementExtraction::FailedCandidateFilter + VARIANT_PRIORITY = { + "consolidated" => 0, + "general" => 1, + "non-consolidated" => 2 + }.freeze + + attr_reader :release, :province, :candidates, :failed_keys, :included_keys, + :approved_covered_keys, :review_pending_covered_keys, :duplicate_slot_keys, :unmatched_keys, + :parser_versions, :failed_extractor_version + + def initialize(release:, province:, candidates:, parser_versions: nil, + failed_extractor_version: Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION) + @release = release + @province = province.to_s.downcase + @candidates = candidates + @parser_versions = Array(parser_versions).compact.map(&:to_s).uniq + @failed_extractor_version = failed_extractor_version.to_s + @candidates_by_key = candidates.group_by { key_for(_1) } + @extractions_by_key = publication_scope.index_by { key_for(_1) } + failed_scope = failure_scope.where(status: "failed") + if self.parser_versions.any? + failed_scope = failed_scope.where( + "llm_response_snapshot ->> 'parser' IN (?)", self.parser_versions + ) + end + @failed_keys = failed_scope.pluck(:asset_sha256, :fiscal_year_end).to_set + approved_slots = publication_scope.where(status: "approved") + .pluck(:institution_canonical_id, :fiscal_year_end) + .map { |institution, fiscal_year_end| [ institution, fiscal_year_end.year ] }.to_set + review_pending_slots = publication_scope.where(status: %w[extracted needs_review]) + .pluck(:institution_canonical_id, :fiscal_year_end) + .map { |institution, fiscal_year_end| [ institution, fiscal_year_end.year ] }.to_set + @unmatched_keys = failed_keys - @candidates_by_key.keys.to_set + @approved_covered_keys = (failed_keys - unmatched_keys).select do |key| + @candidates_by_key.fetch(key).any? { approved_slots.include?(slot_for(_1)) } + end.to_set + @review_pending_covered_keys = (failed_keys - unmatched_keys - approved_covered_keys).select do |key| + @candidates_by_key.fetch(key).any? { review_pending_slots.include?(slot_for(_1)) } + end.to_set + + retryable_keys = failed_keys - unmatched_keys - approved_covered_keys - review_pending_covered_keys + retryable_candidates = candidates.select { retryable_keys.include?(key_for(_1)) } + @winner_by_slot = retryable_candidates.group_by { slot_for(_1) } + .transform_values { select_winner(_1) } + @eligible_document_ids = @winner_by_slot.values.map(&:document_id).to_set + @included_keys = @winner_by_slot.values.map { key_for(_1) }.to_set + @duplicate_slot_keys = retryable_keys - included_keys + end + + def eligible?(candidate) + @eligible_document_ids.include?(candidate.document_id) + end + + def key_for(candidate_or_extraction) + [ candidate_or_extraction.asset_sha256, candidate_or_extraction.fiscal_year_end ] + end + + def report + { + province:, + failed_extractor_version:, + failed_parser_versions: parser_versions.presence, + aggregated_failure_count: failed_keys.length, + included_failure_count: included_keys.length, + eligible_document_count: @eligible_document_ids.length, + public_slot_count: @winner_by_slot.length, + approved_elsewhere_excluded_count: approved_covered_keys.length, + review_pending_elsewhere_excluded_count: review_pending_covered_keys.length, + duplicate_slot_excluded_count: duplicate_slot_keys.length, + unmatched_failure_count: unmatched_keys.length, + reconciled: included_keys.length + approved_covered_keys.length + + review_pending_covered_keys.length + duplicate_slot_keys.length + unmatched_keys.length == + failed_keys.length, + approved_elsewhere_excluded: serialize_keys(approved_covered_keys), + review_pending_elsewhere_excluded: serialize_keys(review_pending_covered_keys), + duplicate_slot_excluded: serialize_duplicate_keys, + unmatched_failures: serialize_keys(unmatched_keys) + } + end + + private + + def publication_scope + release.financial_statement_extractions.where( + extractor_version: Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION + ).where("institution_canonical_id LIKE ?", "ca/#{province}/%") + end + + def failure_scope + release.financial_statement_extractions.where( + extractor_version: failed_extractor_version + ).where("institution_canonical_id LIKE ?", "ca/#{province}/%") + end + + def slot_for(candidate) + [ candidate.institution_canonical_id, candidate.fiscal_year_end.year ] + end + + def select_winner(slot_candidates) + slot_candidates.min_by do |candidate| + variant = candidate.document_canonical_id.split("/").last + [ VARIANT_PRIORITY.fetch(variant, VARIANT_PRIORITY.length), candidate.document_id ] + end + end + + def serialize_keys(keys) + keys.sort_by { |asset_sha256, fiscal_year_end| [ fiscal_year_end, asset_sha256 ] }.map do |key| + asset_sha256, fiscal_year_end = key + { + asset_sha256:, + fiscal_year_end: fiscal_year_end.iso8601, + candidate_institutions: @candidates_by_key.fetch(key, []).map(&:institution_canonical_id).uniq.sort + } + end + end + + def serialize_duplicate_keys + serialize_keys(duplicate_slot_keys).map do |row| + key = [ row.fetch(:asset_sha256), Date.iso8601(row.fetch(:fiscal_year_end)) ] + superseded_slots = @candidates_by_key.fetch(key).filter_map do |candidate| + winner = @winner_by_slot[slot_for(candidate)] + next unless winner + + winner_key = key_for(winner) + { + institution_canonical_id: candidate.institution_canonical_id, + fiscal_year: candidate.fiscal_year_end.year, + winner_document_canonical_id: winner.document_canonical_id, + winner_status: @extractions_by_key[winner_key]&.status + } + end + row.merge(superseded_slots:) + end + end +end diff --git a/app/models/warehouse/financial_statement_extraction/fallback_pipeline.rb b/app/models/warehouse/financial_statement_extraction/fallback_pipeline.rb new file mode 100644 index 00000000..18c31426 --- /dev/null +++ b/app/models/warehouse/financial_statement_extraction/fallback_pipeline.rb @@ -0,0 +1,16 @@ +class Warehouse::FinancialStatementExtraction::FallbackPipeline + def initialize(primary:, fallback:, on:) + @primary = primary + @fallback = fallback + @on = on + end + + def run + @primary.run + rescue => error + raise unless error.is_a?(@on) + + Rails.logger.info("Deterministic financial-statement parser fell back: #{error.message}") + @fallback.run + end +end diff --git a/app/models/warehouse/financial_statement_extraction/number_parser.rb b/app/models/warehouse/financial_statement_extraction/number_parser.rb index 0949e330..651cd88d 100644 --- a/app/models/warehouse/financial_statement_extraction/number_parser.rb +++ b/app/models/warehouse/financial_statement_extraction/number_parser.rb @@ -1,7 +1,21 @@ class Warehouse::FinancialStatementExtraction::NumberParser class ParseError < StandardError; end - NULL_MARKERS = [ "", "-", "—", "–", "nil", "null", "n/a" ].freeze + # Tier 2 evidence for ".": Chapple 2023, asset + # c4b2b35602f5c5eb7fa7794243eff9366c17326c7203c148556231797a8b2d94, PDF page 7 visibly + # leaves the current-year Municipal grants cell blank where OCR emits a lone period. + # Tier 2 evidence for "‒": Frontenac Islands 2011, asset + # 069e092e31ff06f9fe6649f4259701865581e028d6997f08dea70b547c50a375, PDF page 7 visibly + # prints figure dashes in empty numeric cells, matching the embedded text layer. + # Tier 2 evidence for "−": Middlesex 2019, asset + # a091909bd131307b8b9b2d927cdecd51e2d25099f5ac950aa1fc4e041e381e6d, PDF page 6 visibly + # prints standalone minus signs in empty numeric cells while negative values use parentheses. + NULL_MARKERS = [ "", "-", "‐", "‒", "—", "–", "−", "=", ".", '"', "“", "”", "„", "‟", "″", "′′", "nil", "null", "n/a" ].freeze + + def self.null_marker?(raw_text) + normalized = raw_text.to_s.unicode_normalize(:nfkc).tr("\u00A0\u202F", " ").strip.downcase + NULL_MARKERS.include?(normalized) || normalized.match?(/\A[$€£]?\s*[-‐‒—–−=]{1,3}\z/) + end def self.parse(raw_text, raw_label: nil, concept: nil) new(raw_text, raw_label:, concept:).parse @@ -15,12 +29,20 @@ def initialize(raw_text, raw_label: nil, concept: nil) def parse text = @raw_text.unicode_normalize(:nfkc).tr("\u00A0\u202F", " ").strip - raise ParseError, "blank, dash, and zero are distinct; a fact cannot use a null marker" if NULL_MARKERS.include?(text.downcase) + text = repair_leading_ocr_quote(text) + if (match = text.match(/\A\{(?[0-9][0-9,.\s]*)\)\z/)) + text = "(#{match[:numeric]})" + end + raise ParseError, "blank, dash, and zero are distinct; a fact cannot use a null marker" if self.class.null_marker?(text) + validate_signs!(text) - negative = text.match?(/\A\s*\(.*\)\s*\z/) || text.match?(/\A\s*-/) || text.match?(/-\s*\z/) + negative = text.match?(/\A\s*[$€£]?\s*\(.*\)\s*\z/) || text.match?(/\A\s*-/) || text.match?(/-\s*\z/) numeric = text.gsub(/[()$€£]/, "").gsub(/\b(?:cad|can|dollars?|milliers?|millions?)\b/i, "") .gsub(/[+\-]/, "").strip numeric = numeric.gsub(/[[:space:]]/, "") + numeric = repair_ocr_thousands_separator(numeric) + numeric = repair_trailing_ocr_period(numeric) + numeric = repair_ocr_digit_glyphs(numeric) numeric = normalize_separators(numeric) raise ParseError, "not a numeric token: #{@raw_text.inspect}" unless numeric.match?(/\A\d+(?:\.\d+)?\z/) @@ -33,8 +55,56 @@ def parse private + def validate_signs!(text) + signs = text.scan(/[+-]/) + return if signs.empty? + + valid = signs.one? && (text.match?(/\A\s*[$€£]?\s*[+-]/) || text.match?(/[+-]\s*\z/)) + raise ParseError, "not a numeric token: #{@raw_text.inspect}" unless valid + end + + # OCR repairs must match the full token and still pass downstream accounting + # and source checks. Tier 1 removes junk without changing a value, tier 2 + # recognizes nulls, and tier 3 substitutes characters only after visual + # verification of a triggering source. + def repair_leading_ocr_quote(text) + return text unless text.match?(/\A['’][[:space:]]+\d{1,3}(?:,\d{3})+\z/) + + text.sub(/\A['’][[:space:]]+/, "") + end + + def repair_ocr_thousands_separator(numeric) + if numeric.match?(/\A\d{1,3}(?:,\d{3})*,['’],\d{3}(?:,\d{3})*\z/) + return numeric.sub(/,['’],/, ",") + end + # Tier 1 evidence: Neepawa 2010, asset + # 4c4f7da9248bb306742b8dc10bc7b0ed00bcd591d0a6a67bbcdaddec1e6c94cf, PDF page 23 visibly + # prints 20,159 where its embedded text layer contains 20,'159. + if numeric.match?(/\A\d{1,3}(?:,\d{3})*,['’]\d{3}(?:,\d{3})*\z/) + return numeric.sub(/,['’]/, ",") + end + + numeric + end + + def repair_trailing_ocr_period(numeric) + return numeric unless numeric.match?(/\A\d{1,3}(?:,\d{3})+\.\z/) + + numeric.delete_suffix(".") + end + + # Tier 3 evidence: Pelee 2024, asset 74ddb8b64692f591a944b4e02a16aaca748db33b016b049ee5088b35ef176140, + # PDF page 23 visibly prints 11,744 where its text layer contains ll,744. + def repair_ocr_digit_glyphs(numeric) + return numeric unless numeric.match?(/\All(?:,\d{3})+\z/) + + numeric.sub(/\All/, "11") + end + def normalize_separators(numeric) if numeric.include?(",") && numeric.include?(".") + return numeric.delete(",.") if numeric.match?(/\A\d{1,3}(?:[,.]\d{3})+\z/) + decimal_separator = numeric.rindex(",") > numeric.rindex(".") ? "," : "." thousands_separator = decimal_separator == "," ? "." : "," numeric.delete(thousands_separator).sub(decimal_separator, ".") @@ -50,11 +120,14 @@ def normalize_separators(numeric) end def net_debt_label? - @concept == "net_financial_assets" && @raw_label.match?(/\b(?:net debt|dette nette)\b/i) + @concept == "net_financial_assets" && + @raw_label.match?(/\b(?:net debt|dette nette)\b/i) && + !@raw_label.match?(/\b(?:net financial assets|actifs financiers nets)\b/i) end def deficit_label? @concept.in?(%w[annual_surplus accumulated_surplus opening_accumulated_surplus]) && - @raw_label.match?(/\b(?:deficit|déficit)\b/i) + @raw_label.match?(/\b(?:deficit|déficit)\b/i) && + !@raw_label.match?(/\b(?:surplus|exc[eéÉ]dent)\b/i) end end diff --git a/app/models/warehouse/financial_statement_extraction/ocr_text_cache.rb b/app/models/warehouse/financial_statement_extraction/ocr_text_cache.rb new file mode 100644 index 00000000..72472dcf --- /dev/null +++ b/app/models/warehouse/financial_statement_extraction/ocr_text_cache.rb @@ -0,0 +1,174 @@ +require "digest" +require "fileutils" +require "json" +require "tempfile" + +class Warehouse::FinancialStatementExtraction::OcrTextCache + SCHEMA_VERSION = 1 + + class << self + def reset_statistics! + statistics_mutex.synchronize { @statistics = Hash.new(0) } + end + + def record(status) + statistics_mutex.synchronize do + @statistics ||= Hash.new(0) + @statistics[status] += 1 + @statistics.dup + end + end + + private + + def statistics_mutex + @statistics_mutex ||= Mutex.new + end + end + + def initialize(root:, source_path:, reporter: nil) + @root = root.present? ? Pathname(root).expand_path : nil + @source_path = Pathname(source_path).expand_path + @reporter = reporter || ->(event) { warn(event.to_json) } + end + + def fetch(page:, mode:, options:) + return yield unless usable? + + computed = false + computed_text = nil + computing = false + metadata = normalized_metadata(page:, mode:, options:) + key = Digest::SHA256.hexdigest(JSON.generate(metadata)) + started = Process.clock_gettime(Process::CLOCK_MONOTONIC) + if (text = read_entry(key, metadata)) + report("hit", key, metadata, started) + return text + end + + with_lock(key) do + if (text = read_entry(key, metadata)) + report("hit_after_lock", key, metadata, started) + return text + end + + computing = true + text = yield + computing = false + computed = true + computed_text = text + raise ArgumentError, "OCR cache values must be valid UTF-8 strings" unless + text.is_a?(String) && text.encoding == Encoding::UTF_8 && text.valid_encoding? + + write_entry(key, metadata, text) + report("miss", key, metadata, started) + text + end + rescue SystemCallError => error + raise if computing + + report("bypass", nil, { "page" => page, "mode" => mode, "error" => error.message }, started) + computed ? computed_text : yield + end + + private + + def usable? + return false unless @root + + FileUtils.mkdir_p(@root) + @root.directory? && @root.writable? + rescue SystemCallError + false + end + + def normalized_metadata(page:, mode:, options:) + { + "schema_version" => SCHEMA_VERSION, + "source_sha256" => source_sha256, + "page" => Integer(page), + "mode" => mode.to_s, + "options" => deep_stringify_and_sort(options) + } + end + + def source_sha256 + @source_sha256 ||= Digest::SHA256.file(@source_path).hexdigest + end + + def deep_stringify_and_sort(value) + case value + when Hash + value.to_h { |key, child| [ key.to_s, deep_stringify_and_sort(child) ] }.sort.to_h + when Array + value.map { deep_stringify_and_sort(_1) } + when Symbol + value.to_s + else + value + end + end + + def entry_path(key) + @root.join(key.first(2), "#{key}.json") + end + + def lock_path(key) + @root.join(key.first(2), "#{key}.lock") + end + + def read_entry(key, metadata) + payload = JSON.parse(entry_path(key).read) + return unless payload == { + "schema_version" => SCHEMA_VERSION, + "key" => key, + "metadata" => metadata, + "text" => payload["text"] + } + + text = payload["text"] + text if text.is_a?(String) && text.encoding == Encoding::UTF_8 && text.valid_encoding? + rescue Errno::ENOENT, JSON::ParserError, TypeError + nil + end + + def with_lock(key) + path = lock_path(key) + FileUtils.mkdir_p(path.dirname) + File.open(path, File::RDWR | File::CREAT, 0o644) do |lock| + lock.flock(File::LOCK_EX) + yield + ensure + lock.flock(File::LOCK_UN) + end + end + + def write_entry(key, metadata, text) + path = entry_path(key) + FileUtils.mkdir_p(path.dirname) + payload = { + "schema_version" => SCHEMA_VERSION, + "key" => key, + "metadata" => metadata, + "text" => text + } + Tempfile.create([ ".#{key}", ".tmp" ], path.dirname) do |temporary| + temporary.binmode + temporary.write(JSON.generate(payload)) + temporary.flush + temporary.fsync + File.rename(temporary.path, path) + end + end + + def report(status, key, metadata, started) + elapsed = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1_000).round + counts = self.class.record(status) + @reporter.call( + ocr_cache: status, cache_key: key, page: metadata["page"], mode: metadata["mode"], + elapsed_ms: elapsed, counts: + ) + rescue StandardError + nil + end +end diff --git a/app/models/warehouse/financial_statement_extraction/page_locator.rb b/app/models/warehouse/financial_statement_extraction/page_locator.rb index f1abc4a3..297e4512 100644 --- a/app/models/warehouse/financial_statement_extraction/page_locator.rb +++ b/app/models/warehouse/financial_statement_extraction/page_locator.rb @@ -6,20 +6,26 @@ class LocationError < StandardError; end TITLE_PATTERNS = { position: [ /\b(?:consolidated\s+)?statement\s+of\s+financial\s+position\b/i, + /\b(?:consolidated\s+)?statement\s+of\s+financial\s+[a-z]{1,3}\s+position\b/i, /\b(?:état|etat)\s+(?:consolidé|consolide)?\s*(?:de la|de)\s+situation financi(?:è|e)re\b/i ], operations: [ /\b(?:consolidated\s+)?statement\s+of\s+operations(?:\s+and\s+accumulated\s+surplus)?\b/i, + /\b(?:consolidated\s+)?statement\s+of\s+financial\s+activities\b/i, /\b(?:état|etat)\s+(?:consolidé|consolide)?\s*(?:des|de l['’])\s*(?:résultats|resultats|activités financières|activites financieres)\b/i ] }.freeze - EXPLICIT_NON_PRIMARY = /\b(?:schedule|appendix|note|annexe|cédule|cedule)\b/i - CONTENTS = /\b(?:table of contents|contents|table des mati(?:è|e)res|sommaire)\b/i + EXPLICIT_NON_PRIMARY = /\b(?:schedule|appendix|note|annexe|cédule|cedule|exhibit|by fund)\b/i + CONTENTS = /\b(?:table of contents|contents|index to (?:the )?(?:consolidated )?financial statements|table des mati(?:è|e)res|sommaire)\b/i NOTES_HEADING = /\b(?:notes? to (?:the )?(?:consolidated )?financial statements|notes? aux (?:états|etats) financiers)\b/i AUDITOR_HEADING = /\b(?:independent auditors?|auditeurs? ind(?:é|e)pendants?|rapport de l['’]auditeur)\b/i + MALFORMED_OCR_NUMBER = /\d[\d,.]*\.\d{2}[\]\}]/ + OCR_CACHE_POLICY_VERSION = 1 def initialize(pdf_path, pdftotext: "pdftotext", pdfinfo: "pdfinfo", pdftoppm: "pdftoppm", tesseract: "tesseract", - pdfseparate: "pdfseparate", pdfunite: "pdfunite", ghostscript: "gs", max_ocr_pages: 20) + pdfseparate: "pdfseparate", pdfunite: "pdfunite", ghostscript: "gs", max_ocr_pages: 20, + ocr_concurrency: ENV.fetch("MUNICIPAL_FINANCIAL_OCR_CONCURRENCY", 4).to_i, imagemagick: "magick", + ocr_cache_root: ENV["MUNICIPAL_FINANCIAL_OCR_CACHE_ROOT"], ocr_cache_reporter: nil) @pdf_path = Pathname(pdf_path) @pdftotext = pdftotext @pdfinfo = pdfinfo @@ -28,7 +34,15 @@ def initialize(pdf_path, pdftotext: "pdftotext", pdfinfo: "pdfinfo", pdftoppm: " @pdfseparate = pdfseparate @pdfunite = pdfunite @ghostscript = ghostscript + @imagemagick = imagemagick @max_ocr_pages = max_ocr_pages + @ocr_concurrency = [ Integer(ocr_concurrency), 1 ].max + @ocr_dpi = ENV.fetch("MUNICIPAL_FINANCIAL_OCR_DPI", 300).to_i + @page_rotation = 0 + @source_page_landscape = false + @ocr_cache = Warehouse::FinancialStatementExtraction::OcrTextCache.new( + root: ocr_cache_root, source_path: @pdf_path, reporter: ocr_cache_reporter + ) end def locate @@ -36,7 +50,7 @@ def locate page_count = read_page_count page_texts = extract_page_texts(page_count) ocr_pages = [] - if locate_page(page_texts, :position).nil? || locate_page(page_texts, :operations).nil? + if needs_ocr?(page_texts) page_texts, ocr_pages = fill_blank_pages_with_bounded_ocr(page_texts) end @@ -63,13 +77,62 @@ def with_excerpt(pages) end excerpt = File.join(directory, "statement-pages.pdf") stdout, stderr, status = Open3.capture3(@pdfunite, *parts, excerpt) - render_excerpt_with_ghostscript(pages, excerpt) unless status.success? - yield Pathname(excerpt) + if status.success? + normalized_excerpt = File.join(directory, "normalized-statement-pages.pdf") + normalized = normalize_excerpt_with_ghostscript(excerpt, normalized_excerpt) + yield Pathname(normalized ? normalized_excerpt : excerpt) + else + render_excerpt_with_ghostscript(pages, excerpt) + yield Pathname(excerpt) + end end end + def ocr_table_page(page) + page = Integer(page) + @table_ocr_text ||= {} + return @table_ocr_text.fetch(page) if @table_ocr_text.key?(page) + + @table_ocr_text[page] = @ocr_cache.fetch( + page:, mode: "table", options: ocr_cache_options("table") + ) { perform_table_ocr(page) } + end + private + def perform_table_ocr(page) + Dir.mktmpdir("financial-statement-table-ocr") do |directory| + image = render_ocr_image(page, directory:, name: "page", dpi: @ocr_dpi) + thresholded = File.join(directory, "page-thresholded.png") + _stdout, _stderr, status = Open3.capture3( + @imagemagick, image, "-colorspace", "Gray", "-threshold", "45%", thresholded + ) + return ocr_page(page) unless status.success? + + text = run_tesseract(tesseract_args(thresholded, psm: 6, preserve_interword_spaces: true), page:) + normalize_table_ocr_digit_fragments(text) + end + rescue Errno::ENOENT + ocr_page(page) + end + + def needs_ocr?(page_texts) + %i[position operations].any? do |kind| + page = locate_page(page_texts, kind) + page.nil? || !primary_heading?(page_texts.fetch(page).lines.first(12).join, kind:) + end + end + + def normalize_excerpt_with_ghostscript(excerpt, normalized_excerpt) + _stdout, _stderr, status = Open3.capture3( + @ghostscript, "-q", "-dNOPAUSE", "-dBATCH", "-sDEVICE=pdfwrite", + "-sOutputFile=#{normalized_excerpt}", excerpt + ) + status.success? + rescue Errno::ENOENT + false + end + def render_excerpt_with_ghostscript(pages, excerpt) stdout, stderr, status = Open3.capture3( @ghostscript, "-q", "-dNOPAUSE", "-dBATCH", "-sDEVICE=pdfwrite", @@ -91,6 +154,10 @@ def read_page_count match = stdout.match(/^Pages:\s+(\d+)$/) raise LocationError, "pdfinfo did not report a page count" unless match + @page_rotation = stdout[/^Page rot:\s+(\d+)$/, 1].to_i % 360 + if (size = stdout.match(/^Page size:\s+([\d.]+) x ([\d.]+) pts/)) + @source_page_landscape = size[1].to_f > size[2].to_f + end Integer(match[1]) end @@ -110,13 +177,14 @@ def locate_page(page_texts, kind) next unless matches heading = text.lines.first(12).join - [ score(text), page, primary_heading?(heading), heading ] + [ score(text), page, primary_heading?(heading, kind:), heading ] end auditor_page = page_texts.filter_map do |page, text| page if text.lines.first(12).join.match?(AUDITOR_HEADING) end.min primary_after_auditor = matches.select do |score, page, primary, heading| primary && score.positive? && !heading.match?(NOTES_HEADING) && + !heading.match?(AUDITOR_HEADING) && (auditor_page.nil? || page > auditor_page) end return primary_after_auditor.min_by { |_, page, _, _| page }&.at(1) if primary_after_auditor.any? @@ -136,28 +204,162 @@ def score(text) score end - def primary_heading?(heading) - heading.match?(TITLE_PATTERNS.values.flatten.then { |patterns| Regexp.union(patterns) }) + def primary_heading?(heading, kind: nil) + return false if heading.lines.first(4).join.match?(EXPLICIT_NON_PRIMARY) + + patterns = kind ? TITLE_PATTERNS.fetch(kind) : TITLE_PATTERNS.values.flatten + title = Regexp.union(patterns) + heading.lines.any? do |line| + stripped = line.strip + stripped.match?(/\A(?:consolidated\s+statement|statement|état|etat)\b/i) && stripped.match?(title) + end end def fill_blank_pages_with_bounded_ocr(page_texts) pages = page_texts.select { |_, text| text.gsub(/\s/, "").length < 40 }.keys.first(@max_ocr_pages) - pages.each { |page| page_texts[page] = ocr_page(page) } + queue = Queue.new + pages.each { queue << _1 } + results = {} + results_mutex = Mutex.new + errors = Queue.new + workers = [ @ocr_concurrency, pages.length ].min.times.map do + Thread.new do + loop do + page = queue.pop(true) + text = ocr_page(page) + results_mutex.synchronize { results[page] = text } + rescue ThreadError + break + rescue => error + errors << error + break + end + end + end + workers.each(&:join) + raise errors.pop unless errors.empty? + + results.each { |page, text| page_texts[page] = text } [ page_texts, pages ] end def ocr_page(page) + @ocr_cache.fetch(page:, mode: "plain", options: ocr_cache_options("plain")) do + perform_page_ocr(page) + end + end + + def perform_page_ocr(page) Dir.mktmpdir("financial-statement-ocr") do |directory| - output = File.join(directory, "page") - stdout, stderr, status = Open3.capture3(@pdftoppm, "-f", page.to_s, "-l", page.to_s, "-singlefile", "-r", "180", "-png", @pdf_path.to_s, output) - raise LocationError, "pdftoppm page #{page} failed: #{stderr.presence || stdout}" unless status.success? - image = "#{output}.png" - tessdata = Pathname("/Volumes/floppy/york_factory/ocr/tessdata") - args = [ @tesseract, image, "stdout", "-l", "eng+fra" ] - args += [ "--tessdata-dir", tessdata.to_s ] if tessdata.directory? - stdout, stderr, status = Open3.capture3(*args) - raise LocationError, "tesseract page #{page} failed: #{stderr}" unless status.success? - stdout.force_encoding(Encoding::UTF_8).scrub + image = render_ocr_image(page, directory:, name: "page", dpi: @ocr_dpi) + text = run_tesseract(tesseract_args(image), page:) + if text.match?(MALFORMED_OCR_NUMBER) + dense_image = render_ocr_image( + page, directory:, name: "page-dense", dpi: [ @ocr_dpi, 400 ].max + ) + dense_text = run_tesseract(tesseract_args(dense_image, psm: 6), page:) + text = dense_text if malformed_ocr_number_count(dense_text) < malformed_ocr_number_count(text) + end + text end end + + def ocr_cache_options(mode) + @ocr_cache_options ||= {} + orientation_key = [ mode, @page_rotation, @source_page_landscape ] + @ocr_cache_options[orientation_key] ||= { + policy_version: OCR_CACHE_POLICY_VERSION, + dpi: @ocr_dpi, + page_rotation: @page_rotation, + source_page_landscape: @source_page_landscape, + pdftoppm: executable_signature(@pdftoppm), + tesseract: executable_signature(@tesseract), + tessdata: tessdata_signature, + language: "eng+fra", + mode_options: mode == "table" ? { + imagemagick: executable_signature(@imagemagick), threshold: "45%", psm: 6, + preserve_interword_spaces: true, digit_fragment_normalizer: 1 + } : { + psm: nil, dense_retry_dpi: [ @ocr_dpi, 400 ].max, + dense_retry_psm: 6, malformed_number_pattern: MALFORMED_OCR_NUMBER.source + } + } + end + + def executable_signature(command) + path = if Pathname(command).absolute? + Pathname(command) + else + ENV.fetch("PATH", "").split(File::PATH_SEPARATOR) + .map { Pathname(_1).join(command) }.find(&:executable?) + end + return { command:, resolved: nil } unless path&.file? + + stat = path.stat + { command:, resolved: path.realpath.to_s, size: stat.size, mtime_ns: stat.mtime.nsec, + mtime: stat.mtime.to_i } + rescue SystemCallError + { command:, resolved: nil } + end + + def tessdata_signature + root = Pathname("/Volumes/floppy/york_factory/ocr/tessdata") + %w[eng fra].to_h do |language| + path = root.join("#{language}.traineddata") + next [ language, nil ] unless path.file? + + stat = path.stat + [ language, { path: path.to_s, size: stat.size, mtime: stat.mtime.to_i, mtime_ns: stat.mtime.nsec } ] + end + end + + def render_ocr_image(page, directory:, name:, dpi:) + output = File.join(directory, name) + stdout, stderr, status = Open3.capture3( + @pdftoppm, "-f", page.to_s, "-l", page.to_s, "-singlefile", + "-r", dpi.to_s, "-png", @pdf_path.to_s, output + ) + raise LocationError, "pdftoppm page #{page} failed: #{stderr.presence || stdout}" unless status.success? + + correct_ocr_orientation("#{output}.png", directory:, name:) + end + + def correct_ocr_orientation(image, directory:, name:) + correction = (360 - @page_rotation) % 360 + return image unless correction.in?([ 90, 270 ]) + return image if @source_page_landscape + + oriented = File.join(directory, "#{name}-oriented.png") + _stdout, _stderr, status = Open3.capture3( + @imagemagick, image, "-rotate", correction.to_s, oriented + ) + status.success? ? oriented : image + rescue Errno::ENOENT + image + end + + def tesseract_args(image, psm: nil, preserve_interword_spaces: false) + args = [ @tesseract, image, "stdout", "-l", "eng+fra" ] + tessdata = Pathname("/Volumes/floppy/york_factory/ocr/tessdata") + args += [ "--tessdata-dir", tessdata.to_s ] if tessdata.directory? + args += [ "--psm", psm.to_s ] if psm + args += [ "-c", "preserve_interword_spaces=1" ] if preserve_interword_spaces + args + end + + def run_tesseract(args, page:) + stdout, stderr, status = Open3.capture3(*args) + raise LocationError, "tesseract page #{page} failed: #{stderr}" unless status.success? + + stdout.force_encoding(Encoding::UTF_8).scrub + end + + def malformed_ocr_number_count(text) + text.scan(MALFORMED_OCR_NUMBER).length + end + + def normalize_table_ocr_digit_fragments(text) + text.gsub(/(? "total_financial_assets_single_component", + "total_liabilities" => "total_liabilities_single_component", + "total_non_financial_assets" => "total_non_financial_assets_single_component" + }.freeze Result = Data.define(:status, :facts, :checks, :prompt, :response, :locator_result, :language, :statement_basis) class ResponseError < StandardError; end + def self.normalize_column_year(raw_value, **) = raw_value.to_s.strip + + def self.single_component_concepts(response) + SINGLE_COMPONENT_FLAGS.filter_map { |concept, flag| concept if response.fetch(flag, false) } + end + def initialize(pdf_path:, institution_canonical_id:, institution_name:, document_canonical_id:, asset_sha256:, fiscal_year_end:, population: nil, model: DEFAULT_MODEL, llm_client: nil, page_locator: nil) @@ -25,7 +36,7 @@ def initialize(pdf_path:, institution_canonical_id:, institution_name:, document def run verify_source_hash! - locator_result = @page_locator.locate + locator_result = primary_statement_locator(@page_locator.locate) prompt = build_prompt(locator_result) raw_response = nil response = nil @@ -34,6 +45,57 @@ def run response = raw_response.respond_to?(:content) ? raw_response.content : raw_response end response = JSON.parse(response) if response.is_a?(String) + result_from(response, locator_result, prompt:) + rescue JSON::ParserError, KeyError, ArgumentError, ResponseError, Timeout::Error => error + raise ResponseError, error.message + end + + def revalidate(response:, prompt: nil, source_pages: nil) + verify_source_hash! + response = JSON.parse(response) if response.is_a?(String) + locator_result = primary_statement_locator(@page_locator.locate) + response = remap_excerpt_pages(response, locator_result, source_pages) if source_pages + result_from(response, locator_result, prompt:) + rescue JSON::ParserError, KeyError, ArgumentError, ResponseError => error + raise ResponseError, error.message + end + + private + + def primary_statement_locator(locator_result) + pages = [ locator_result.position_page, locator_result.operations_page ].compact.uniq + supporting = headline_supporting_page(locator_result, excluding: pages) + pages << supporting if supporting + raise ResponseError, "primary financial statements were not located" if pages.empty? + + locator_result.with(candidate_pages: pages.sort) + end + + def headline_supporting_page(locator_result, excluding:) + locator_result.page_texts.filter_map do |page, text| + next if page.in?(excluding) + + heading = text.lines.first(20).join + next unless heading.match?(/schedule|appendix|annexe|cédule|cedule/i) + next unless text.match?(/revenues?|revenus|produits/i) && text.match?(/expenses?|expenditures?|dépenses|depenses|charges/i) + next unless text.match?(/(?= 3 } + [ page, numeric_cells ] + end.max_by { |page, score| [ score, -page ] }&.first + end + + def remap_excerpt_pages(response, locator_result, source_pages) + response = response.deep_dup + response.fetch("facts").each do |fact| + source_page = source_pages[fact.fetch("concept")] + excerpt_index = locator_result.candidate_pages.index(source_page) + fact["excerpt_page"] = excerpt_index + 1 if excerpt_index + end + response + end + + def result_from(response, locator_result, prompt:) validate_response!(response, locator_result) facts = normalize_facts(response.fetch("facts"), locator_result) validator = Warehouse::FinancialStatementExtraction::Validator.new( @@ -42,19 +104,16 @@ def run flags: { remeasurement_present: response.fetch("remeasurement_present"), operations_adjustment_present: response.fetch("operations_adjustment_present"), - rollforward_adjustment_present: response.fetch("rollforward_adjustment_present") + rollforward_adjustment_present: response.fetch("rollforward_adjustment_present"), + single_component_concepts: self.class.single_component_concepts(response) } ) checks = [ source_identity_check, *validator.validate ] status = validator.acceptable?(checks) ? "extracted" : "needs_review" Result.new(status:, facts:, checks:, prompt:, response:, locator_result:, language: response.fetch("language"), statement_basis: response.fetch("statement_basis")) - rescue JSON::ParserError, KeyError, ArgumentError, ResponseError, Timeout::Error => error - raise ResponseError, error.message end - private - def source_identity_check { id: "source_identity", @@ -93,7 +152,12 @@ def build_prompt(locator_result) Rules: - Extract only the expected fiscal year's ACTUAL column. Never use budget or comparative columns. - Return only totals explicitly printed in the attached primary statements. Never calculate or infer a missing fact. + - A supporting schedule may supply a consolidated total revenue or total expenses that is absent from the main operations statement. - A section total may be printed on an unlabeled line immediately before the next heading. In that case, use the section heading (for example "Revenues") as raw_label and the printed total as raw_text. + - When a financial-assets, liabilities, or non-financial-assets section contains exactly one printed component and no + separate total, return that component as the corresponding section total. Preserve the component's exact raw_label + and raw_text, set confidence no higher than 0.90, and set the matching *_single_component boolean true. + Otherwise every *_single_component boolean must be false. - Preserve raw_label and raw_text exactly as printed. - scale is exactly 1, 1000, or 1000000 and must follow the printed heading. - Preserve parentheses and minus signs in raw_text. The deterministic parser applies signs and scale. @@ -134,10 +198,16 @@ def validate_response!(response, locator_result) confidence = Float(fact.fetch("confidence")) raise ResponseError, "confidence out of range" unless confidence.between?(0, 1) end + self.class.single_component_concepts(response).each do |concept| + fact = response.fetch("facts").find { _1.fetch("concept") == concept } + raise ResponseError, "single-component flag lacks #{concept}" unless fact + raise ResponseError, "single-component confidence exceeds 0.90" if Float(fact.fetch("confidence")) > 0.90 + end end def normalize_facts(raw_facts, locator_result) raw_facts.map do |fact| + source_page = locator_result.candidate_pages.fetch(Integer(fact.fetch("excerpt_page")) - 1) { concept: fact.fetch("concept"), statement: fact.fetch("statement"), @@ -147,8 +217,11 @@ def normalize_facts(raw_facts, locator_result) fact.fetch("raw_text"), raw_label: fact.fetch("raw_label"), concept: fact.fetch("concept") ) * Integer(fact.fetch("scale")), scale: Integer(fact.fetch("scale")), - source_page: locator_result.candidate_pages.fetch(Integer(fact.fetch("excerpt_page")) - 1), - column_year: fact.fetch("column_year"), + source_page:, + column_year: self.class.normalize_column_year( + fact.fetch("column_year"), fiscal_year: @fiscal_year_end.year, + page_text: locator_result.page_texts.fetch(source_page) + ), extraction_confidence: BigDecimal(fact.fetch("confidence").to_s) } end diff --git a/app/models/warehouse/financial_statement_extraction/prairie_failed_candidate_filter.rb b/app/models/warehouse/financial_statement_extraction/prairie_failed_candidate_filter.rb new file mode 100644 index 00000000..92ed6e8c --- /dev/null +++ b/app/models/warehouse/financial_statement_extraction/prairie_failed_candidate_filter.rb @@ -0,0 +1,3 @@ +class Warehouse::FinancialStatementExtraction::PrairieFailedCandidateFilter < + Warehouse::FinancialStatementExtraction::FailedCandidateFilter +end diff --git a/app/models/warehouse/financial_statement_extraction/processor.rb b/app/models/warehouse/financial_statement_extraction/processor.rb new file mode 100644 index 00000000..b4c0d81d --- /dev/null +++ b/app/models/warehouse/financial_statement_extraction/processor.rb @@ -0,0 +1,105 @@ +class Warehouse::FinancialStatementExtraction::Processor + RERUN_POLICIES = %w[missing failed review all].freeze + DETAIL_HEADLINE_STATUSES = %w[extracted needs_review approved].freeze + Outcome = Data.define(:status, :stage, :extraction_id, :error) + + def initialize(release:, rerun: "missing") + @release = release + @rerun = rerun.to_s + raise ArgumentError, "unsupported rerun policy #{@rerun.inspect}" unless @rerun.in?(RERUN_POLICIES) + end + + def call(candidate) + detailed = extraction_for(candidate, Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION) + return outcome("skipped", "detailed", detailed.id) unless runnable?(detailed) + return outcome("missing_asset", "source") unless candidate.pdf_path.file? + + headline = extraction_for(candidate, Warehouse::FinancialStatementExtraction::Pipeline::EXTRACTOR_VERSION) + headline = run_headline(candidate, headline) if runnable?(headline) + return outcome(headline.status, "headline", headline.id) unless headline.status.in?(DETAIL_HEADLINE_STATUSES) + + detailed = run_detailed(candidate, detailed, headline) + outcome(detailed.status, "detailed", detailed.id) + rescue ActiveRecord::RecordNotUnique + outcome("concurrent_skip", "identity") + rescue => error + outcome("failed", "exception", nil, "#{error.class}: #{error.message}") + end + + private + + def extraction_for(candidate, version) + @release.financial_statement_extractions.find_or_initialize_by( + asset_sha256: candidate.asset_sha256, + extractor_version: version, + fiscal_year_end: candidate.fiscal_year_end + ) do |extraction| + extraction.assign_attributes(base_attributes(candidate, version:)) + end + end + + def run_headline(candidate, extraction) + return extraction if extraction.persisted? && !runnable?(extraction) + + extraction.assign_attributes(base_attributes( + candidate, version: Warehouse::FinancialStatementExtraction::Pipeline::EXTRACTOR_VERSION + )) + extraction.save! + extraction.extractor.extract( + pdf_path: candidate.pdf_path, + institution_name: candidate.institution_name, + population: candidate.population + ) + extraction.reload + end + + def run_detailed(candidate, extraction, headline) + extraction.assign_attributes(base_attributes( + candidate, version: Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION, + language: headline.language, statement_basis: headline.statement_basis + )) + extraction.save! + extraction.extractor.extract_detailed( + pdf_path: candidate.pdf_path, + institution_name: candidate.institution_name, + population: candidate.population + ) + extraction.reload + end + + def base_attributes(candidate, version:, language: nil, statement_basis: "consolidated") + { + institution_canonical_id: candidate.institution_canonical_id, + document_canonical_id: candidate.document_canonical_id, + fiscal_year_end: candidate.fiscal_year_end, + statement_basis:, + language:, + llm_model: model_for(version), + status: "pending" + } + end + + def model_for(version) + if version == Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION + Warehouse::FinancialStatementExtraction::DetailedPipeline::DEFAULT_MODEL + else + Warehouse::FinancialStatementExtraction::Pipeline::DEFAULT_MODEL + end + end + + def runnable?(extraction) + return true unless extraction.persisted? + return false if extraction.reviewed_at? + + case @rerun + when "missing" then extraction.status == "pending" + when "failed" then extraction.status.in?(%w[pending failed]) + when "review" then extraction.status.in?(%w[pending failed needs_review]) + when "all" then true + end + end + + def outcome(status, stage, extraction_id = nil, error = nil) + Outcome.new(status:, stage:, extraction_id:, error:) + end +end diff --git a/app/models/warehouse/financial_statement_extraction/quebec_form_pipeline.rb b/app/models/warehouse/financial_statement_extraction/quebec_form_pipeline.rb new file mode 100644 index 00000000..8f145fff --- /dev/null +++ b/app/models/warehouse/financial_statement_extraction/quebec_form_pipeline.rb @@ -0,0 +1,438 @@ +class Warehouse::FinancialStatementExtraction::QuebecFormPipeline + PARSER_VERSION = "quebec-mamh-form-v1" + Result = Warehouse::FinancialStatementExtraction::DetailedPipeline::Result + Unsupported = Class.new(StandardError) + Headline = Data.define(:pipeline) do + def run = pipeline.run_headline + end + + OPERATIONS_TOTAL_ROWS = { + total_revenue: 13, + total_expenses: 24, + annual_surplus: 25, + opening_accumulated_surplus: 28, + accumulated_surplus: 29 + }.freeze + POSITION_ROWS_BY_YEAR = { + (..2019) => { + total_financial_assets: 8, total_liabilities: 15, + net_financial_assets: 16, total_non_financial_assets: 21, + accumulated_surplus: 22 + }, + 2020 => { + total_financial_assets: 8, total_liabilities: 16, + net_financial_assets: 17, total_non_financial_assets: 22, + accumulated_surplus: 23 + }, + (2021..) => { + total_financial_assets: 8, total_liabilities: 16, + net_financial_assets: 17, total_non_financial_assets: 23, + accumulated_surplus: 24 + } + }.freeze + REVENUE_ROWS = (1..12) + EXPENSE_ROWS = (14..23) + SECTION_LABELS = { + total_revenue: "Revenus", + total_expenses: "Charges", + total_financial_assets: "ACTIFS FINANCIERS", + total_liabilities: "PASSIFS", + total_non_financial_assets: "ACTIFS NON FINANCIERS" + }.freeze + + def self.applicable?(institution_canonical_id:, fiscal_year_end:) + institution_canonical_id.start_with?("ca/qc/") && fiscal_year_end.year >= 2016 + end + + def initialize(pdf_path:, institution_canonical_id:, document_canonical_id:, asset_sha256:, + fiscal_year_end:, population: nil, page_locator: nil, **) + @pdf_path = Pathname(pdf_path) + @institution_canonical_id = institution_canonical_id + @document_canonical_id = document_canonical_id + @asset_sha256 = asset_sha256 + @fiscal_year = fiscal_year_end.year + @population = population + @page_locator = page_locator || Warehouse::FinancialStatementExtraction::PageLocator.new( + @pdf_path, max_ocr_pages: 0 + ) + end + + def run + verify_source! + located = prefer_mamh_form_pages(@page_locator.locate) + operations = FormPage.new(located.page_texts.fetch(located.operations_page), @fiscal_year, current_column: 1) + position = FormPage.new(located.page_texts.fetch(located.position_page), @fiscal_year, current_column: 0) + operations, position, located = repair_mamh_table_ocr(operations, position, located) + @scale = Warehouse::FinancialStatementExtraction::ScaleDetector.detect( + located.page_texts.values_at(located.operations_page, located.position_page) + ) + facts = build_facts(operations, position, located) + line_items = build_line_items(operations, located.operations_page) + flags = flags_for(operations, position) + validator = Warehouse::FinancialStatementExtraction::Validator.new( + facts:, line_items:, fiscal_year: @fiscal_year, population: @population, + page_texts: located.page_texts, flags: + ) + checks = [ source_check, *validator.validate ] + raise Unsupported, failed_checks(checks) unless validator.acceptable?(checks) + + response = { + "parser" => PARSER_VERSION, + "headline" => flags.stringify_keys, + "details" => { + "source" => "Quebec standardized municipal financial report", + "ocr_pages" => located.ocr_pages + } + } + Result.new( + status: "extracted", facts:, line_items:, checks:, + prompt: nil, response:, locator_result: located, + language: "fr", statement_basis: "consolidated" + ) + rescue Warehouse::FinancialStatementExtraction::PageLocator::LocationError, + Warehouse::FinancialStatementExtraction::NumberParser::ParseError, + KeyError, ArgumentError => error + raise Unsupported, error.message + end + + def run_headline + detailed = run + flags = detailed.response.fetch("headline") + Warehouse::FinancialStatementExtraction::Pipeline::Result.new( + status: detailed.status, facts: detailed.facts, checks: detailed.checks, + prompt: nil, response: flags.merge( + "parser" => PARSER_VERSION, + "fiscal_year" => @fiscal_year, + "language" => detailed.language, + "statement_basis" => detailed.statement_basis + ), + locator_result: detailed.locator_result, language: detailed.language, + statement_basis: detailed.statement_basis + ) + end + + private + + def prefer_mamh_form_pages(located) + operations_pages = located.page_texts.filter_map do |page, text| + page if text.match?(/Rapport financier.{0,30}\bS7\b/i) + end + return located unless operations_pages.one? + + operations_page = operations_pages.first + position_page = operations_page + 1 + return located unless located.page_texts.key?(position_page) + + candidates = [ operations_page - 1, operations_page, position_page, position_page + 1 ] + .select { _1.between?(1, located.page_count) }.uniq.sort + located.with(operations_page:, position_page:, candidate_pages: candidates) + end + + def verify_source! + raise Unsupported, "missing source PDF" unless @pdf_path.file? + actual = Digest::SHA256.file(@pdf_path).hexdigest + raise Unsupported, "asset SHA mismatch" unless actual == @asset_sha256 + end + + def repair_mamh_table_ocr(operations, position, located) + missing_operations = OPERATIONS_TOTAL_ROWS.values.reject { operations[_1] } + missing_position = position_rows.values.reject { position[_1] } + return [ operations, position, located ] if missing_operations.empty? && missing_position.empty? + + operations_ocr = @page_locator.ocr_table_page(located.operations_page) + position_ocr = @page_locator.ocr_table_page(located.position_page) + ocr_operations = OcrFormPage.new(operations_ocr, @fiscal_year, current_column: 1) + ocr_position = OcrFormPage.new(position_ocr, @fiscal_year, current_column: 0) + @ocr_operation_line_items = { + "revenue" => ocr_operations.section_rows(/\ARevenus\b/i, /\ACharges\b/i), + "expense" => ocr_operations.section_rows(/\ACharges\b/i, /\AExc[eé]dent\b/i) + } + + repair_operation_rows(operations, ocr_operations, missing_operations) + repair_position_rows(position, ocr_position, missing_position) + remaining = OPERATIONS_TOTAL_ROWS.values.reject { operations[_1] } + + position_rows.values.reject { position[_1] } + raise Unsupported, "required MAMH rows missing after table OCR: #{remaining.join(', ')}" if remaining.any? + + page_texts = located.page_texts.merge( + located.operations_page => [ located.page_texts.fetch(located.operations_page), operations_ocr ].join("\n"), + located.position_page => [ located.page_texts.fetch(located.position_page), position_ocr ].join("\n") + ) + repaired = located.with( + page_texts:, + ocr_pages: (located.ocr_pages + [ located.operations_page, located.position_page ]).uniq.sort + ) + [ operations, position, repaired ] + end + + def repair_operation_rows(page, ocr, missing) + repairs = { + 13 => ocr.section_total(/\ARevenus\b/i, /\ACharges\b/i, label: "Revenus"), + 24 => ocr.section_total(/\ACharges\b/i, /\AExc[eé]dent\b/i, label: "Charges"), + 25 => ocr.label(/\AExc[eé]dent.*(?:de l['’]exercice|li[eé] aux activit[eé]s)/i), + 28 => ocr.label(/\ASolde redress[eé]\b/i), + 29 => ocr.label(/(?:fin de l['’]exercice|\AExc[eé]dent.*accumul[eé].*fin)/i) + } + missing.each { |row| page.put(row, repairs[row]) if repairs[row] } + end + + def repair_position_rows(page, ocr, missing) + rows = position_rows + repairs = { + rows.fetch(:total_financial_assets) => ocr.section_total( + /\AACTIFS FINANCIERS\b/i, /\APASSIFS\b/i, label: "ACTIFS FINANCIERS" + ), + rows.fetch(:total_liabilities) => ocr.section_total( + /\APASSIFS\b/i, /\AACTIFS FINANCIERS NETS|DETTE NETTE/i, label: "PASSIFS" + ), + rows.fetch(:net_financial_assets) => ocr.label(/ACTIFS FINANCIERS NETS|DETTE NETTE/i), + rows.fetch(:total_non_financial_assets) => ocr.section_total( + /\AACTIFS NON FINANCIERS\b/i, /EXC[EÉ]DENT.*ACCUMUL[EÉ]/i, label: "ACTIFS NON FINANCIERS" + ), + rows.fetch(:accumulated_surplus) => ocr.label(/\AEXC[EÉ]DENT.*ACCUMUL[EÉ]/i) + } + missing.each { |row| page.put(row, repairs[row]) if repairs[row] } + end + + def source_check + { id: "source_identity", status: "pass", detail: "document=#{@document_canonical_id}; asset_sha256=#{@asset_sha256}" } + end + + def build_facts(operations, position, located) + facts = [] + OPERATIONS_TOTAL_ROWS.each do |concept, row| + entry = operations.fetch(row) + facts << fact(concept, entry, located.operations_page, + concept.in?(%i[opening_accumulated_surplus accumulated_surplus]) ? "accumulated_surplus" : "operations") + end + position_rows.each do |concept, row| + entry = position.fetch(row) + facts.reject! { |item| item[:concept] == concept.to_s } + facts << fact(concept, entry, located.position_page, "financial_position") + end + facts + end + + def fact(concept, entry, source_page, statement) + raw_label = SECTION_LABELS.fetch(concept, entry.fetch(:label)) + { + concept: concept.to_s, statement:, raw_label:, + raw_text: entry.fetch(:raw_text), + value: parse(entry.fetch(:raw_text), raw_label:, concept: concept.to_s) * @scale, + scale: @scale, source_page:, column_year: entry.fetch(:column_year), + extraction_confidence: BigDecimal("1") + } + end + + def build_line_items(operations, source_page) + return build_ocr_line_items(source_page) if @ocr_operation_line_items + + positions = Hash.new(0) + [ [ "revenue", REVENUE_ROWS ], [ "expense", EXPENSE_ROWS ] ].flat_map do |flow, rows| + rows.filter_map do |row| + entry = operations[row] + next unless entry && entry[:raw_text].present? + + label = entry.fetch(:label) + next if label.blank? + result = { + flow:, category: label, label:, raw_text: entry.fetch(:raw_text), + value: parse(entry.fetch(:raw_text)) * @scale, scale: @scale, source_page:, + column_year: entry.fetch(:column_year), position: positions[flow], + extraction_confidence: BigDecimal("1") + } + positions[flow] += 1 + result + end + end + end + + def build_ocr_line_items(source_page) + @ocr_operation_line_items.flat_map do |flow, rows| + rows.map.with_index do |entry, position| + label = entry.fetch(:label) + { + flow:, category: label, label:, raw_text: entry.fetch(:raw_text), + value: parse(entry.fetch(:raw_text)) * @scale, scale: @scale, source_page:, + column_year: entry.fetch(:column_year), position:, + extraction_confidence: BigDecimal("1") + } + end + end + end + + def flags_for(operations, position) + { + remeasurement_present: position.value_present?(26), + operations_adjustment_present: false, + rollforward_adjustment_present: operations.value_present?(27) + } + end + + def position_rows + POSITION_ROWS_BY_YEAR.find do |year_or_range, _| + year_or_range == @fiscal_year || year_or_range.respond_to?(:cover?) && year_or_range.cover?(@fiscal_year) + end.last + end + + def parse(raw_text, raw_label: nil, concept: nil) + Warehouse::FinancialStatementExtraction::NumberParser.parse(raw_text, raw_label:, concept:) + end + + def failed_checks(checks) + failures = checks.select { |check| check[:status] == "fail" }.map { |check| check[:id] } + "Quebec form did not pass deterministic validation: #{failures.join(', ')}" + end + + class FormPage + NUMBER = /\(?-?\d[\d ,.\u00A0\u202F]*\)?/ + + def initialize(text, fiscal_year, current_column:) + @text = text + @fiscal_year = fiscal_year + @preferred_current_column = current_column + @column_starts = locate_columns + @rows = parse_rows + end + + def fetch(row) = @rows.fetch(row) + def [](row) = @rows[row] + def value_present?(row) = @rows[row]&.fetch(:raw_text).present? + def put(row, entry) = @rows[row] ||= entry + + private + + def locate_columns + candidates = @text.lines.filter_map do |line| + current_year_starts = [] + line.to_enum(:scan, /(? start_index && @lines[_1].match?(finish_pattern) } + return unless finish_index + + line = @lines[(start_index + 1)...finish_index].reverse.find { value(_1) } + entry(line, label) if line + end + + def section_rows(start_pattern, finish_pattern) + start_index = @lines.index { _1.match?(start_pattern) } + return [] unless start_index + + finish_index = @lines.each_index.find { _1 > start_index && @lines[_1].match?(finish_pattern) } + return [] unless finish_index + + @lines[(start_index + 1)...finish_index].filter_map { row_entry(_1) } + end + + private + + def entry(line, label) + { label:, raw_text: value(line), column_year: @fiscal_year.to_s } + end + + def value(line) + tokens = line.to_s.scan(NUMBER) + return if tokens.length < @expected_columns + + tokens.last(@expected_columns).fetch(@current_column) + end + + def row_entry(line) + matches = line.to_enum(:scan, NUMBER).map do + [ Regexp.last_match[0], Regexp.last_match.begin(0) ] + end + return if matches.length < @expected_columns + + columns = matches.last(@expected_columns) + label = line[0...columns.first.second].to_s.sub(/\s+\d{1,2}\s*\z/, "").squish + return if label.blank? || label.match?(/\A\d+\z/) + + { label:, raw_text: columns.fetch(@current_column).first, column_year: @fiscal_year.to_s } + end + end +end diff --git a/app/models/warehouse/financial_statement_extraction/quebec_form_processor.rb b/app/models/warehouse/financial_statement_extraction/quebec_form_processor.rb new file mode 100644 index 00000000..2285c5dc --- /dev/null +++ b/app/models/warehouse/financial_statement_extraction/quebec_form_processor.rb @@ -0,0 +1,139 @@ +class Warehouse::FinancialStatementExtraction::QuebecFormProcessor + Outcome = Data.define(:status, :document_canonical_id, :detailed_extraction_id, :error) + + def initialize(release:) + @release = release + end + + def call(candidate) + return outcome("missing_asset", candidate, nil, "archived PDF is missing") unless candidate.pdf_path.file? + unless pipeline_class.applicable?( + institution_canonical_id: candidate.institution_canonical_id, + fiscal_year_end: candidate.fiscal_year_end + ) + return outcome("unsupported", candidate, nil, "deterministic form parser does not apply") + end + + detailed = extraction(candidate, Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION) + return outcome("skipped", candidate, detailed.id) if detailed.reviewed_at? + if detailed.persisted? && detailed.status == "extracted" && + detailed.llm_response_snapshot&.fetch("parser", nil) == parser_version + return outcome("skipped", candidate, detailed.id) + end + + result = pipeline_class.new( + pdf_path: candidate.pdf_path, + institution_canonical_id: candidate.institution_canonical_id, + institution_name: candidate.institution_name, + document_canonical_id: candidate.document_canonical_id, + asset_sha256: candidate.asset_sha256, + fiscal_year_end: candidate.fiscal_year_end, + population: candidate.population + ).run + headline = extraction(candidate, Warehouse::FinancialStatementExtraction::Pipeline::EXTRACTOR_VERSION) + persist(candidate, headline, detailed, result) + outcome("extracted", candidate, detailed.id) + rescue => error + if error.is_a?(pipeline_class::Unsupported) + record_failure(candidate, detailed, error, check_id: "deterministic_parser", parser_outcome: "unsupported") + outcome("unsupported", candidate, detailed&.id, error.message) + else + record_failure(candidate, detailed, error, check_id: "parser_execution", parser_outcome: "failed") + outcome("failed", candidate, detailed&.id, "#{error.class}: #{error.message}") + end + end + + private + + def extraction(candidate, version) + @release.financial_statement_extractions.find_or_initialize_by( + asset_sha256: candidate.asset_sha256, + extractor_version: version, + fiscal_year_end: candidate.fiscal_year_end + ) do |row| + row.assign_attributes( + institution_canonical_id: candidate.institution_canonical_id, + document_canonical_id: candidate.document_canonical_id, + statement_basis: "consolidated", + language: nil, + llm_model: parser_version, + status: "pending" + ) + end + end + + def persist(candidate, headline, detailed, result) + flags = result.response.fetch("headline") + @release.transaction do + [ headline, detailed ].each do |row| + raise ArgumentError, "reviewed extraction is immutable" if row.reviewed_at? + + row.assign_attributes( + institution_canonical_id: candidate.institution_canonical_id, + document_canonical_id: candidate.document_canonical_id, + asset_sha256: candidate.asset_sha256, + fiscal_year_end: candidate.fiscal_year_end, + statement_basis: result.statement_basis, + language: result.language, + llm_model: parser_version, + status: "extracted", check_results: result.checks, + llm_prompt_snapshot: { prompt: nil }, error_message: nil + ) + row.llm_response_snapshot = response_for(row, result, flags) + row.save! + Warehouse::FinancialStatementFact.where(financial_statement_extraction: row).delete_all + result.facts.each { |attributes| row.financial_statement_facts.create!(attributes) } + end + Warehouse::FinancialStatementLineItem.where(financial_statement_extraction: detailed).delete_all + result.line_items.each { |attributes| detailed.financial_statement_line_items.create!(attributes) } + end + end + + def response_for(extraction, result, flags) + if extraction.extractor_version == Warehouse::FinancialStatementExtraction::Pipeline::EXTRACTOR_VERSION + flags.merge( + "parser" => parser_version, + "fiscal_year" => extraction.fiscal_year_end.year, + "language" => result.language, + "statement_basis" => result.statement_basis + ) + else + result.response + end + end + + def record_failure(candidate, extraction, error, check_id:, parser_outcome:) + return unless extraction + return if extraction.reviewed_at? + if extraction.persisted? && extraction.status == "extracted" && + extraction.llm_response_snapshot&.fetch("parser", nil) == parser_version + return + end + + extraction.assign_attributes( + institution_canonical_id: candidate.institution_canonical_id, + document_canonical_id: candidate.document_canonical_id, + asset_sha256: candidate.asset_sha256, + fiscal_year_end: candidate.fiscal_year_end, + statement_basis: "consolidated", + language: nil, + llm_model: parser_version, + status: "failed", + check_results: [ { id: check_id, status: "fail", detail: error.message } ], + llm_prompt_snapshot: { prompt: nil }, + llm_response_snapshot: { "parser" => parser_version, "outcome" => parser_outcome }, + error_message: "#{error.class}: #{error.message}" + ) + extraction.save! + rescue ActiveRecord::RecordNotUnique + nil + end + + def outcome(status, candidate, extraction_id = nil, error = nil) + Outcome.new(status:, document_canonical_id: candidate.document_canonical_id, + detailed_extraction_id: extraction_id, error:) + end + + def pipeline_class = Warehouse::FinancialStatementExtraction::QuebecFormPipeline + def parser_version = pipeline_class::PARSER_VERSION +end diff --git a/app/models/warehouse/financial_statement_extraction/response_schema.rb b/app/models/warehouse/financial_statement_extraction/response_schema.rb index 6d904620..bd56f409 100644 --- a/app/models/warehouse/financial_statement_extraction/response_schema.rb +++ b/app/models/warehouse/financial_statement_extraction/response_schema.rb @@ -7,6 +7,9 @@ class Warehouse::FinancialStatementExtraction::ResponseSchema < RubyLLM::Schema boolean :remeasurement_present boolean :operations_adjustment_present boolean :rollforward_adjustment_present + boolean :total_financial_assets_single_component + boolean :total_liabilities_single_component + boolean :total_non_financial_assets_single_component array :facts, min_items: 1, max_items: 9 do object do string :concept, enum: Warehouse::FinancialStatementFact::CONCEPTS diff --git a/app/models/warehouse/financial_statement_extraction/reviewer.rb b/app/models/warehouse/financial_statement_extraction/reviewer.rb new file mode 100644 index 00000000..feec65de --- /dev/null +++ b/app/models/warehouse/financial_statement_extraction/reviewer.rb @@ -0,0 +1,263 @@ +require "digest" + +class Warehouse::FinancialStatementExtraction::Reviewer + REVIEWER = "deterministic-source-reaudit-v1" + VISUAL_REVIEWER = "deterministic-plus-visual-reaudit-v1" + DETERMINISTIC_REVIEWERS = [ REVIEWER, VISUAL_REVIEWER ].freeze + GENERIC_TABLE_OCR_PAGE_LIMIT = 12 + DEFAULT_ASSET_ROOT = Warehouse::FinancialStatementExtraction::CandidateSet::DEFAULT_ASSET_ROOT + Result = Data.define(:status, :checks) + + def initialize(extraction:, asset_root: ENV.fetch("PUBLIC_INSTITUTION_ASSET_ROOT", DEFAULT_ASSET_ROOT.to_s), + page_locator: nil, visual_model: Warehouse::FinancialStatementExtraction::VisualEvidenceReviewer::DEFAULT_MODEL, + visual_llm_client: nil) + @extraction = extraction + @asset_root = Pathname(asset_root).expand_path + @page_locator = page_locator + @visual_model = visual_model + @visual_llm_client = visual_llm_client + end + + def review! + validate_reviewable! + result = audit + @extraction.with_lock do + return result if @extraction.reviewed_at? + + validate_reviewable! + @extraction.update!(check_results: result.checks) + if result.status == "approved" + @extraction.approve!( + reviewer: deterministic_reviewer(result.checks), + notes: deterministic_review_notes(result.checks) + ) + else + @extraction.update!(status: "needs_review", error_message: "independent deterministic re-audit failed") + end + end + result + end + + def reaudit! + validate_reauditable! + result = audit + return result unless result.status == "approved" + + previous_reviewer = @extraction.reviewed_by + previous_reviewed_at = @extraction.reviewed_at + previous_checks = @extraction.check_results.deep_dup + previous_digest = Digest::SHA256.hexdigest(JSON.generate(previous_checks)) + provenance = "previous reviewer=#{previous_reviewer} at #{previous_reviewed_at.iso8601}; " \ + "#{previous_checks.length} checks sha256=#{previous_digest}" + @extraction.transaction do + @extraction.update!( + check_results: result.checks, + reviewed_by: deterministic_reviewer(result.checks), + reviewed_at: Time.current, + review_notes: "#{deterministic_review_notes(result.checks)}; legacy provenance: #{provenance}" + ) + end + result + end + + def audit + validate_detailed! + document, asset, pdf_path = source + source_check = verify_source(document, asset, pdf_path) + if source_check.fetch(:status) == "fail" + return Result.new(status: "needs_review", checks: [ source_check ]) + end + locator = @page_locator || Warehouse::FinancialStatementExtraction::PageLocator.new( + pdf_path, max_ocr_pages: review_ocr_page_limit + ) + located = align_parser_pages(locator.locate) + located = enrich_parser_table_ocr(located, locator) + audit_validator = validator(located) + validation_checks = audit_validator.validate + scale_check = source_scale_check(located) + checks = [ source_check, scale_check, *validation_checks ] + unless audit_validator.acceptable?(validation_checks) + checks = Warehouse::FinancialStatementExtraction::VisualEvidenceReviewer.new( + extraction: @extraction, page_locator: locator, model: @visual_model, + llm_client: @visual_llm_client + ).apply(checks) + end + status = if scale_check.fetch(:status) == "pass" && audit_validator.acceptable?(checks) + "approved" + else + "needs_review" + end + + Result.new(status:, checks:) + end + + private + + def validate_reviewable! + validate_detailed! + unless @extraction.status.in?(%w[extracted needs_review]) + raise ArgumentError, "extraction must be extracted or awaiting review" + end + raise ArgumentError, "reviewed extraction is immutable" if @extraction.reviewed_at? + end + + def validate_reauditable! + validate_detailed! + raise ArgumentError, "only approved extractions can be re-audited" unless @extraction.status == "approved" + unless @extraction.reviewed_at? && @extraction.reviewed_by? + raise ArgumentError, "approved extraction must have existing review provenance" + end + if @extraction.reviewed_by.in?(DETERMINISTIC_REVIEWERS) + raise ArgumentError, "extraction already has current deterministic review provenance" + end + end + + def validate_detailed! + unless @extraction.extractor_version == Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION + raise ArgumentError, "only detailed extractions can be reviewed" + end + end + + def deterministic_reviewer(checks) + visual = checks.any? { _1[:id].start_with?("visual_evidence:") && _1[:status] == "pass" } + visual ? VISUAL_REVIEWER : REVIEWER + end + + def deterministic_review_notes(checks) + visual = deterministic_reviewer(checks) == VISUAL_REVIEWER + "Source hash, units, page evidence, raw parsing, accounting identities, and line sums " \ + "independently revalidated#{'; blind visual source transcription also matched' if visual}" + end + + def source + document = @extraction.institution_release.institution_documents.find_by!( + canonical_id: @extraction.document_canonical_id + ) + asset = document.institution_document_assets.find_by!(content_sha256: @extraction.asset_sha256) + pdf_path = @asset_root.join(asset.archive_path).expand_path + unless pdf_path.to_s.start_with?("#{@asset_root}/") + raise ArgumentError, "asset path escapes root" + end + [ document, asset, pdf_path ] + end + + def verify_source(document, asset, pdf_path) + return check("source_identity", "fail", "archived PDF is missing") unless pdf_path.file? + + actual = Digest::SHA256.file(pdf_path).hexdigest + status = actual == asset.content_sha256 ? "pass" : "fail" + check( + "source_identity", status, + "document=#{document.canonical_id}; expected=#{asset.content_sha256}; actual=#{actual}" + ) + end + + def validator(located) + Warehouse::FinancialStatementExtraction::Validator.new( + facts: @extraction.financial_statement_facts.map { fact_attributes(_1) }, + line_items: @extraction.financial_statement_line_items.map { line_item_attributes(_1) }, + fiscal_year: @extraction.fiscal_year_end.year, + page_texts: located.page_texts, + flags: audited_flags(located) + ) + end + + def headline_response + Hash(@extraction.llm_response_snapshot).fetch("headline", {}) + end + + def review_ocr_page_limit + case Hash(@extraction.llm_response_snapshot)["parser"] + when Warehouse::FinancialStatementExtraction::QuebecFormPipeline::PARSER_VERSION then 0 + when *Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline::REVIEWABLE_PARSER_VERSIONS then 8 + else 20 + end + end + + def align_parser_pages(located) + parser = Hash(@extraction.llm_response_snapshot)["parser"] + return located unless parser == Warehouse::FinancialStatementExtraction::QuebecFormPipeline::PARSER_VERSION + + operations_page = @extraction.financial_statement_facts.find_by(concept: "total_revenue")&.source_page + position_page = @extraction.financial_statement_facts.find_by(concept: "total_financial_assets")&.source_page + return located unless operations_page && position_page + + candidates = [ operations_page - 1, operations_page, position_page, position_page + 1 ] + .select { _1.between?(1, located.page_count) }.uniq.sort + located.with(operations_page:, position_page:, candidate_pages: candidates) + end + + def enrich_parser_table_ocr(located, locator) + parser = Hash(@extraction.llm_response_snapshot)["parser"] + pages = if parser.in?( + Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline::REVIEWABLE_PARSER_VERSIONS + ) + [ located.position_page, located.operations_page ].uniq + elsif parser == Warehouse::FinancialStatementExtraction::QuebecFormPipeline::PARSER_VERSION + Array(Hash(@extraction.llm_response_snapshot).dig("details", "ocr_pages")).map { Integer(_1) } + else + (@extraction.financial_statement_facts.pluck(:source_page) + + @extraction.financial_statement_line_items.pluck(:source_page)) + .uniq.sort.select { _1.between?(1, located.page_count) && located.page_texts.key?(_1) } + .first(GENERIC_TABLE_OCR_PAGE_LIMIT) + end + return located if pages.empty? + + replacements = pages.to_h do |page| + [ page, [ located.page_texts.fetch(page), locator.ocr_table_page(page) ].join("\n") ] + end + located.with( + page_texts: located.page_texts.merge(replacements), + ocr_pages: (located.ocr_pages + pages).uniq.sort + ) + end + + def source_scale_check(located) + expected = Warehouse::FinancialStatementExtraction::ScaleDetector.detect( + located.page_texts.values_at(located.operations_page, located.position_page) + ) + stored = (@extraction.financial_statement_facts.pluck(:scale) + + @extraction.financial_statement_line_items.pluck(:scale)).uniq + status = stored == [ expected ] ? "pass" : "fail" + check("source_scale", status, "source=#{expected}; stored=#{stored.sort.join(',')}") + end + + def audited_flags(located) + facts = @extraction.financial_statement_facts.index_by(&:concept) + position_text = located.page_texts.fetch(located.position_page) + operations_text = located.page_texts.fetch(located.operations_page) + cited_text = @extraction.financial_statement_facts.pluck(:source_page).uniq + .map { located.page_texts[_1].to_s }.join("\n") + operations_discrepancy = if facts.values_at("total_revenue", "total_expenses", "annual_surplus").all? + facts["total_revenue"].value - facts["total_expenses"].value - facts["annual_surplus"].value + end + rollforward_discrepancy = if facts.values_at("opening_accumulated_surplus", "annual_surplus", "accumulated_surplus").all? + facts["opening_accumulated_surplus"].value + facts["annual_surplus"].value - facts["accumulated_surplus"].value + end + { + remeasurement_present: position_text.match?(/remeasurement|remesurement|réévaluation|reevaluation/i), + operations_adjustment_present: operations_discrepancy&.nonzero? && + operations_text.match?(/capital(?:-related)? (?:contributions|transfers)|contributed|donated tangible capital|transfers?(?: related to| relating to| for) capital|gain|loss|restructur/i), + rollforward_adjustment_present: rollforward_discrepancy&.nonzero? && + cited_text.match?(/remeasurement|remesurement|restat|retrait|adjust|redress|other comprehensive.{0,30}(?:income|loss)|réévaluation|reevaluation/i), + single_component_concepts: + Warehouse::FinancialStatementExtraction::Pipeline.single_component_concepts(headline_response) + } + end + + def fact_attributes(fact) + fact.attributes.symbolize_keys.slice( + :concept, :value, :raw_text, :raw_label, :scale, :statement, + :source_page, :column_year, :extraction_confidence + ) + end + + def line_item_attributes(item) + item.attributes.symbolize_keys.slice( + :flow, :category, :label, :value, :raw_text, :scale, + :source_page, :column_year, :position, :extraction_confidence + ) + end + + def check(id, status, detail) = { id:, status:, detail: } +end diff --git a/app/models/warehouse/financial_statement_extraction/saskatchewan_form_pipeline.rb b/app/models/warehouse/financial_statement_extraction/saskatchewan_form_pipeline.rb new file mode 100644 index 00000000..4fc3d46f --- /dev/null +++ b/app/models/warehouse/financial_statement_extraction/saskatchewan_form_pipeline.rb @@ -0,0 +1,545 @@ +class Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline + PARSER_VERSION = "prairie-municipal-form-v3" + REVIEWABLE_PARSER_VERSIONS = [ + PARSER_VERSION, "prairie-municipal-form-v2", "prairie-municipal-form-v1", + "saskatchewan-municipal-form-v1" + ].freeze + Unsupported = Class.new(StandardError) + Result = Warehouse::FinancialStatementExtraction::DetailedPipeline::Result + Headline = Data.define(:pipeline) do + def run = pipeline.run_headline + end + EXPENSE_HEADING_PATTERN = /\A(?:EXPENSES?|EXPENDITURES)\b/i + + FACT_LABELS = { + total_financial_assets: /\A(?:Total )?Financial Assets\b/i, + total_liabilities: /\ATotal\s+Liabi\s*lities\b/i, + net_financial_assets: /\ANET (?:FINANCIAL ASSETS(?: \(DEBT\))?|DEBT)\b/i, + total_non_financial_assets: /\A(?:Total )?Non-Financial Assets\b/i, + accumulated_surplus: /\AACCUMULATED SURPLUS(?: \(DEFICIT\))?/i, + total_revenue: /\ATotal Revenues?\b/i, + total_expenses: /\ATotal (?:Operating )?(?:Expenses?|Expenditures)\b/i, + opening_accumulated_surplus: /\AAccumulated Surplus(?: \(Deficit\))?,?\s+(?:at )?Beginning of Year\b/i + }.freeze + ANNUAL_SURPLUS_PATTERN = /\A(?:Annual Surplus|Surplus \(Deficit\) of Revenues over (?:Expenses?|Expenditures)|Excess \((?:Deficiency|Shortfall)\) of Revenues? over Expenses?|Excess of (?:Revenue over Expenses?|Expenses? over Revenue)|Excess revenues \(expenses\)).*\z/i + POSITION_SECTIONS = { + total_financial_assets: [ /\AFINANCIAL ASSETS\b/i, /\A(?:FINANCIAL )?LIABI\s*LITIES\b/i ], + total_liabilities: [ /\A(?:FINANCIAL )?LIABI\s*LITIES\b/i, /\A(?:NET (?:FINANCIAL ASSETS|DEBT)|NON-FINANCIAL ASSETS)\b/i ], + total_non_financial_assets: [ /\ANON-FINANCIAL ASSETS\b/i, /\AACCUMULATED SURPLUS\b/i ] + }.freeze + POSITION_SECTION_LABELS = { + total_financial_assets: "FINANCIAL ASSETS", + total_liabilities: "LIABILITIES", + total_non_financial_assets: "NON-FINANCIAL ASSETS" + }.freeze + + def self.applicable?(institution_canonical_id:, **) + institution_canonical_id.start_with?("ca/sk/", "ca/ab/", "ca/bc/", "ca/ns/") + end + + def initialize(pdf_path:, institution_canonical_id:, document_canonical_id:, asset_sha256:, + fiscal_year_end:, population: nil, page_locator: nil, **) + @pdf_path = Pathname(pdf_path) + @institution_canonical_id = institution_canonical_id + @document_canonical_id = document_canonical_id + @asset_sha256 = asset_sha256 + @fiscal_year = fiscal_year_end.year + @population = population + @page_locator = page_locator || Warehouse::FinancialStatementExtraction::PageLocator.new( + @pdf_path, max_ocr_pages: 8 + ) + end + + def run + verify_source! + located = enrich_with_table_ocr(@page_locator.locate) + parse_located(located) + rescue Unsupported => error + raise error unless located && !located.ocr_pages.include?(located.operations_page) + + parse_located(with_operations_table_ocr(located)) + rescue Warehouse::FinancialStatementExtraction::PageLocator::LocationError => error + raise Unsupported, error.message + end + + def run_headline + detailed = run + flags = detailed.response.fetch("headline") + Warehouse::FinancialStatementExtraction::Pipeline::Result.new( + status: detailed.status, facts: detailed.facts, checks: detailed.checks, + prompt: nil, response: flags.merge( + "parser" => PARSER_VERSION, "fiscal_year" => @fiscal_year, + "language" => detailed.language, "statement_basis" => detailed.statement_basis + ), + locator_result: detailed.locator_result, language: detailed.language, + statement_basis: detailed.statement_basis + ) + end + + private + + def parse_located(located) + operations = EnglishPage.new(located.page_texts.fetch(located.operations_page), fiscal_year: @fiscal_year, kind: :operations) + position = EnglishPage.new(located.page_texts.fetch(located.position_page), fiscal_year: @fiscal_year, kind: :position) + @scale = Warehouse::FinancialStatementExtraction::ScaleDetector.detect( + located.page_texts.values_at(located.operations_page, located.position_page) + ) + facts = build_facts(operations, position, located) + fallbacks = facts.filter_map do |fact| + confidence = fact.fetch(:extraction_confidence) + next unless confidence < 1 + + { + "concept" => fact.fetch(:concept), + "type" => confidence == BigDecimal("0.90") ? "single_component" : "unlabeled_section_total" + } + end + line_items = build_line_items(operations, located.operations_page) + flags = flags_for(operations, position, facts) + validator = Warehouse::FinancialStatementExtraction::Validator.new( + facts:, line_items:, fiscal_year: @fiscal_year, population: @population, + page_texts: located.page_texts, flags: + ) + checks = [ source_check, *validator.validate ] + if fallbacks.any? { _1.fetch("type") == "single_component" } + position_surplus = checks.find { _1[:id] == "position_surplus" } + checks << { + id: "position_single_component", + status: position_surplus&.fetch(:status) == "pass" ? "pass" : "fail", + detail: "single-component total requires a passing, non-skipped position surplus identity" + } + end + failures = checks.select { _1[:status] == "fail" }.map { _1[:id] } + raise Unsupported, "municipal form failed deterministic validation: #{failures.join(', ')}" unless validator.acceptable?(checks) + + response = { + "parser" => PARSER_VERSION, + "headline" => flags.stringify_keys, + "details" => { + "source" => "standardized Canadian municipal financial statement", + "position_total_fallbacks" => fallbacks + } + } + Result.new( + status: "extracted", facts:, line_items:, checks:, prompt: nil, response:, + locator_result: located, language: "en", statement_basis: "consolidated" + ) + rescue Warehouse::FinancialStatementExtraction::NumberParser::ParseError, + KeyError, ArgumentError => error + raise Unsupported, error.message + end + + def with_operations_table_ocr(located) + page = located.operations_page + located.with( + page_texts: located.page_texts.merge(page => @page_locator.ocr_table_page(page)), + ocr_pages: (located.ocr_pages + [ page ]).uniq.sort + ) + rescue Warehouse::FinancialStatementExtraction::PageLocator::LocationError => error + raise Unsupported, error.message + end + + def verify_source! + raise Unsupported, "missing source PDF" unless @pdf_path.file? + raise Unsupported, "asset SHA mismatch" unless Digest::SHA256.file(@pdf_path).hexdigest == @asset_sha256 + end + + def enrich_with_table_ocr(located) + required = { + located.position_page => FACT_LABELS.values_at( + :total_financial_assets, :total_liabilities, :net_financial_assets, + :total_non_financial_assets, :accumulated_surplus + ), + located.operations_page => [ + FACT_LABELS.fetch(:total_revenue), FACT_LABELS.fetch(:total_expenses), annual_surplus_pattern + ] + } + replacements = required.filter_map do |page, _patterns| + text = located.page_texts.fetch(page) + if page == located.position_page + next if position_values_complete?(text) + + table_text = @page_locator.ocr_table_page(page) + next unless position_values_complete?(table_text) + + next [ page, table_text ] + end + next if operations_values_complete?(text) + + table_text = @page_locator.ocr_table_page(page) + next unless operations_values_complete?(table_text) + + [ page, table_text ] + end.to_h + return located if replacements.empty? + + located.with( + page_texts: located.page_texts.merge(replacements), + ocr_pages: (located.ocr_pages + replacements.keys).uniq.sort + ) + end + + def position_values_complete?(text) + page = EnglishPage.new(text, fiscal_year: @fiscal_year, kind: :position) + FACT_LABELS.slice( + :total_financial_assets, :total_liabilities, :net_financial_assets, + :total_non_financial_assets, :accumulated_surplus + ).all? do |concept, pattern| + entry = page.find(pattern) + entry = nil if entry && null?(entry[:current]) + entry ||= position_total(page, concept) + entry.present? + end + rescue Unsupported, ArgumentError + false + end + + def operations_values_complete?(text) + page = EnglishPage.new(text, fiscal_year: @fiscal_year, kind: :operations) + totals = FACT_LABELS.slice(:total_revenue, :total_expenses).map do |concept, pattern| + entry = page.find(pattern) + entry = nil if entry && null?(entry[:current]) + entry || operation_total(page, concept) + end + entries = [ *totals, annual_surplus_entry(page) ] + return false if entries.any?(&:nil?) + + entries.all? do |entry| + current = entry[:current] + next false if current.blank? || null?(current) + + parse(current) + true + end + rescue Unsupported, ArgumentError, Warehouse::FinancialStatementExtraction::NumberParser::ParseError + false + end + + def source_check + { id: "source_identity", status: "pass", detail: "document=#{@document_canonical_id}; asset_sha256=#{@asset_sha256}" } + end + + def build_facts(operations, position, located) + facts = FACT_LABELS.filter_map do |concept, pattern| + page = concept.in?(%i[total_financial_assets total_liabilities net_financial_assets total_non_financial_assets accumulated_surplus]) ? position : operations + statement = if concept.in?(%i[opening_accumulated_surplus accumulated_surplus]) + "accumulated_surplus" + elsif page == position + "financial_position" + else + "operations" + end + entry = page.find(pattern) + entry = nil if entry && null?(entry[:current]) + entry ||= operation_total(operations, concept) if page == operations + entry ||= position_total(position, concept) if page == position + next if concept == :opening_accumulated_surplus && entry.nil? + + raise Unsupported, "row not found: #{pattern.inspect}" unless entry + fact(concept, entry, page == position ? located.position_page : located.operations_page, statement) + end + annual = annual_surplus_entry(operations) + raise Unsupported, "final annual surplus row not found" unless annual + + facts << fact(:annual_surplus, annual, located.operations_page, "operations") + facts + end + + def operation_total(operations, concept) + case concept + when :total_revenue + row = operations.section(/\AREVENUES?\b/i, EXPENSE_HEADING_PATTERN) + .reverse.find { _1[:label].blank? && _1[:current].present? && !null?(_1[:current]) } + heading = operations.find(/\AREVENUES?\b/i)&.fetch(:label) + row&.merge(label: heading || "Revenue", extraction_confidence: BigDecimal("0.95")) + when :total_expenses + rows = operations.section(EXPENSE_HEADING_PATTERN, annual_surplus_pattern) + row = rows.reverse.find { _1[:label].blank? && _1[:current].present? && !null?(_1[:current]) } + heading = operations.find(EXPENSE_HEADING_PATTERN)&.fetch(:label) + row&.merge(label: heading || "Expenses", extraction_confidence: BigDecimal("0.95")) + end + end + + def annual_surplus_entry(operations) + rows = operations.lines.reverse.select { _1[:label].match?(annual_surplus_pattern) } + rows.find { _1[:current].present? && !null?(_1[:current]) } || rows.first + end + + def position_total(position, concept) + boundaries = POSITION_SECTIONS[concept] + return unless boundaries + + rows = position.section(*boundaries) + component_rows = rows.select do |row| + row[:label].to_s.match?(/[[:alnum:]]/) && row[:current].present? && !null?(row[:current]) + end + row = rows.reverse.find do |candidate| + !candidate[:label].to_s.match?(/[[:alnum:]]/) && + candidate[:current].present? && !null?(candidate[:current]) + end + if component_rows.length >= 2 && row + return row.merge( + label: POSITION_SECTION_LABELS.fetch(concept), + extraction_confidence: BigDecimal("0.95") + ) + end + return unless concept == :total_non_financial_assets && component_rows.one? && row.nil? + + component_rows.first.merge(extraction_confidence: BigDecimal("0.90")) + end + + def fact(concept, entry, source_page, statement) + raw_label = entry.fetch(:label) + raw_text = entry.fetch(:current) + { + concept: concept.to_s, statement:, raw_label:, raw_text:, + value: parse(raw_text, raw_label:, concept: concept.to_s) * @scale, scale: @scale, + source_page:, column_year: entry.fetch(:column_year), + extraction_confidence: entry.fetch(:extraction_confidence, BigDecimal("1")) + } + rescue Warehouse::FinancialStatementExtraction::NumberParser::ParseError => error + raise Unsupported, "#{concept}: #{error.message}" + end + + def build_line_items(operations, source_page) + revenue = operations.section(/\AREVENUES?\b/i, EXPENSE_HEADING_PATTERN) + expenses = operations.section( + EXPENSE_HEADING_PATTERN, + Regexp.union(/\ATotal (?:Operating )?(?:Expenses?|Expenditures)\b/i, annual_surplus_pattern) + ) + revenue.reject! { _1[:label].blank? || _1[:label].match?(/\ATotal Revenues?\b/i) } + expenses.reject! { _1[:label].blank? || _1[:label].match?(/\ATotal (?:Expenses?|Expenditures)\b/i) } + revenue |= capital_rows(operations) + positions = Hash.new(0) + [ [ "revenue", revenue ], [ "expense", expenses ] ].flat_map do |flow, rows| + rows.filter_map do |entry| + next if entry[:current].blank? || null?(entry[:current]) + + label = entry.fetch(:label) + item = { + flow:, category: label, label:, raw_text: entry.fetch(:current), + value: parse(entry.fetch(:current)) * @scale, scale: @scale, source_page:, + column_year: entry.fetch(:column_year), position: positions[flow], + extraction_confidence: entry.fetch(:extraction_confidence, BigDecimal("1")) + } + positions[flow] += 1 + item + end + end + end + + def flags_for(operations, position, facts) + capital = capital_rows(operations).first + facts_by_concept = facts.index_by { _1.fetch(:concept) } + operations_discrepancy = facts_by_concept.fetch("total_revenue").fetch(:value) - + facts_by_concept.fetch("total_expenses").fetch(:value) - + facts_by_concept.fetch("annual_surplus").fetch(:value) + remeasurement = position.lines.any? do |line| + line[:label].match?(/remeasurement/i) && line[:current].present? && !null?(line[:current]) + end + { + remeasurement_present: remeasurement, + operations_adjustment_present: operations_discrepancy.nonzero? && + capital && capital[:current].present? && !null?(capital[:current]), + rollforward_adjustment_present: remeasurement + } + end + + def null?(value) = Warehouse::FinancialStatementExtraction::NumberParser.null_marker?(value) + + def capital_rows(operations) + operations.lines.select do |line| + line[:label].match?(/\A(?:.*Capital Grants and Contributions|Other Capital Contributions|.*transfers for capital|Contributed assets)\b/i) && + line[:current].present? && !null?(line[:current]) + end + end + + def annual_surplus_pattern + ANNUAL_SURPLUS_PATTERN + end + + def parse(raw_text, raw_label: nil, concept: nil) + Warehouse::FinancialStatementExtraction::NumberParser.parse(raw_text, raw_label:, concept:) + end + + class EnglishPage + NUMBER = /\(?-?(?:\d \d{1,2}[,.]\d{3}|\d{1,3}(?:[,.]\d{3})+|\d+)\)?|(? start_index && lines[index][:label].match?(finish_pattern) + end + raise Unsupported, "section total not found: #{finish_pattern.inspect}" unless finish_index + + lines[(start_index + 1)...finish_index] + end + + private + + def locate_columns(text) + candidates = text.lines.filter_map do |line| + years = line.scan(/(?= 2 + + budget = @kind == :operations && line.match?(/budget|fiscal plan/i) + expected = years.length + (budget && years.count(@fiscal_year.to_s) == 1 ? 1 : 0) + current = if budget || years.count(@fiscal_year.to_s) > 1 + 1 + else + years.index(@fiscal_year.to_s) + end + [ expected, current, @fiscal_year.to_s, line ] + end + selected = candidates.max_by { |expected, _, _, line| [ expected, line.length ] } + unless selected + years = text.lines.filter_map do |line| + stripped = line.strip + stripped if stripped.match?(/\A(?:19|20)\d{2}\z/) + end + if years.include?(@fiscal_year.to_s) && years.length >= 2 + selected = [ years.length, years.index(@fiscal_year.to_s), @fiscal_year.to_s ] + end + end + selected ||= infer_standard_columns(text) + raise Unsupported, "current-year column layout not found" unless selected + + selected.first(3) + end + + def infer_standard_columns(text) + title = @kind == :operations ? /Statement of Operations/i : /Statement of Financial Position/i + return unless text.match?(title) && text.match?(/(?= 2 + end + numeric if numeric.in?([ 2, 3 ]) + end + expected, occurrences = counts.tally.max_by { |columns, frequency| [ frequency, columns ] } + return unless occurrences.to_i >= 3 + return if @kind == :position && expected != 2 + + current = @kind == :operations && expected == 3 ? 1 : 0 + [ expected, current, @fiscal_year.to_s ] + end + + def parse_lines(text) + pending_labels = [] + text.lines.filter_map do |raw_line| + if raw_line.strip.blank? + pending_labels.clear + next + end + + row = parse_line(raw_line.chomp) + unless row + pending_labels.clear + next + end + if row[:current].blank? + label = row.fetch(:label) + if FACT_LABELS.values.any? { label.match?(_1) } + pending_labels.replace([ label ]) + else + pending_labels << label + end + pending_labels.shift while pending_labels.length > 3 + next row + end + + if row.fetch(:label).blank? && + Warehouse::FinancialStatementExtraction::NumberParser.null_marker?(row.fetch(:current)) + next row + end + + combined_label = [ *pending_labels, row.fetch(:label) ].join(" ").squish + pending_labels.clear + if wrapped_label?(combined_label, current_label: row.fetch(:label)) + row.merge(label: combined_label) + else + row + end + end + end + + def wrapped_label?(label, current_label:) + label.match?(ANNUAL_SURPLUS_PATTERN) || + (current_label.blank? && FACT_LABELS.values.any? { label.match?(_1) }) + end + + def parse_line(line) + stripped = line.strip + return if stripped.blank? + return if column_header?(stripped) + if stripped.match?(/\A(?:REVENUES?|EXPENSES?|EXPENDITURES)(?:\s+\(unaudited\))?\z/i) + return { label: stripped, current: nil, column_year: @column_year } + end + + scannable = stripped.gsub(/\([^)]*(?:schedule|note)[^)]*\)/i) { " " * _1.length } + matches = scannable.to_enum(:scan, NUMBER).map { [ Regexp.last_match[0], Regexp.last_match.begin(0) ] } + numeric = matches.select { |token, _| token.in?(%w[- =]) || token.count("0-9") >= 2 } + return { label: stripped, current: nil, column_year: @column_year } if numeric.length < 2 + + return { label: stripped, current: nil, column_year: @column_year } if numeric.length < @expected_columns + + if numeric.length == @expected_columns + 1 + previous_token, previous_start = numeric[-2] + fragment, fragment_start = numeric[-1] + separator = stripped[(previous_start + previous_token.length)...fragment_start] + if fragment.count("0-9").between?(1, 2) && + !Warehouse::FinancialStatementExtraction::NumberParser.null_marker?(fragment) && + !fragment.match?(/\A(?:19|20)\d{2}\z/) && separator.match?(/\A[,.]\z/) + numeric.pop + end + end + + while numeric.length > @expected_columns && + Warehouse::FinancialStatementExtraction::NumberParser.null_marker?(numeric.last.first) + surviving = numeric[-(@expected_columns + 1), @expected_columns] + break unless surviving&.none? do |token, _position| + Warehouse::FinancialStatementExtraction::NumberParser.null_marker?(token) || + token.match?(/\A(?:19|20)\d{2}\z/) + end + + numeric.pop + end + + columns = numeric.last(@expected_columns) + label_end = columns.first.last + { label: stripped[0...label_end].strip, current: columns.fetch(@current_column).first, column_year: @column_year } + end + + def column_header?(text) + remainder = text + .gsub(/(? error + check("line_raw_parse:#{line_item_key(item)}", "fail", error.message) + end + + def line_item_column_year_check(item) + label = item.fetch(:column_year).to_s + valid = label.match?(/(?= 0.8 ? "pass" : "fail", confidence ? format("minimum fact confidence %.3f", confidence) : "no facts") end @@ -178,6 +267,34 @@ def normalized_number(value) value.to_s.unicode_normalize(:nfkc).tr("\u00A0\u202F", " ").gsub(/[[:space:],.]/, "").gsub(/[^0-9()\-]/, "") end + def normalized_without_numbers(value) + normalized(value).gsub(/\b\d+(?: \d+)*\b/, " ").squish + end + + def effective_flag?(flag) + concepts, discrepancy = case flag + when :remeasurement_present + [ %w[net_financial_assets total_non_financial_assets accumulated_surplus], + ->(net, non_financial, surplus) { net + non_financial - surplus } ] + when :operations_adjustment_present + [ %w[total_revenue total_expenses annual_surplus], + ->(revenue, expenses, surplus) { revenue - expenses - surplus } ] + when :rollforward_adjustment_present + [ %w[opening_accumulated_surplus annual_surplus accumulated_surplus], + ->(opening, annual, closing) { opening + annual - closing } ] + else + return false + end + return false unless flags[flag] + + selected = concepts.map { by_concept[_1] } + return false if selected.any?(&:nil?) + + values = selected.map { decimal(_1.fetch(:value)) } + tolerance = selected.map { Integer(_1.fetch(:scale)) }.max + discrepancy.call(*values).abs > tolerance + end + def check(id, status, detail) { id:, status:, detail: } end diff --git a/app/models/warehouse/financial_statement_extraction/visual_evidence_response_schema.rb b/app/models/warehouse/financial_statement_extraction/visual_evidence_response_schema.rb new file mode 100644 index 00000000..fa3101f6 --- /dev/null +++ b/app/models/warehouse/financial_statement_extraction/visual_evidence_response_schema.rb @@ -0,0 +1,15 @@ +require "ruby_llm/schema" + +class Warehouse::FinancialStatementExtraction::VisualEvidenceResponseSchema < RubyLLM::Schema + array :claims, min_items: 1 do + object do + string :id + boolean :found + string :transcribed_label + string :transcribed_category + string :raw_text + string :column_year + integer :excerpt_page + end + end +end diff --git a/app/models/warehouse/financial_statement_extraction/visual_evidence_reviewer.rb b/app/models/warehouse/financial_statement_extraction/visual_evidence_reviewer.rb new file mode 100644 index 00000000..b068294b --- /dev/null +++ b/app/models/warehouse/financial_statement_extraction/visual_evidence_reviewer.rb @@ -0,0 +1,145 @@ +class Warehouse::FinancialStatementExtraction::VisualEvidenceReviewer + DEFAULT_MODEL = ENV.fetch("MUNICIPAL_FINANCIAL_VISUAL_REVIEW_MODEL", "claude-haiku-4-5-20251001") + + def initialize(extraction:, page_locator:, model: DEFAULT_MODEL, llm_client: nil) + @extraction = extraction + @page_locator = page_locator + @model = model + @llm_client = llm_client || method(:call_ruby_llm) + end + + def apply(checks) + claims = claims_for(checks) + return checks unless eligible?(checks, claims) + + pages = claims.values.map { _1.fetch(:physical_page) }.uniq.sort + page_map = pages.each_with_index.to_h { |page, index| [ page, index + 1 ] } + prompt = build_prompt(claims, page_map) + response = @page_locator.with_excerpt(pages) do |excerpt| + raw = @llm_client.call(prompt:, pdf_path: excerpt.to_s) + raw.respond_to?(:content) ? raw.content : raw + end + response = JSON.parse(response) if response.is_a?(String) + apply_response(checks, claims, page_map, response) + rescue => error + [ *checks, check("visual_evidence:verifier", "fail", "#{error.class}: #{error.message}") ] + end + + private + + def eligible?(checks, claims) + failures = checks.select { _1[:status] == "fail" } + failures.any? && failures.all? { claims.key?(_1[:id]) } + end + + def claims_for(checks) + failed_ids = checks.select { _1[:status] == "fail" }.pluck(:id) + claims = {} + @extraction.financial_statement_facts.each do |fact| + id = "evidence:#{fact.concept}" + next unless id.in?(failed_ids) + + claims[id] = { + id:, kind: "fact", physical_page: fact.source_page, label: fact.raw_label, + category: "", value: fact.value, scale: fact.scale, concept: fact.concept + } + end + @extraction.financial_statement_line_items.each do |item| + id = "line_evidence:#{line_item_key(item)}" + next unless id.in?(failed_ids) + + claims[id] = { + id:, kind: "line_item", physical_page: item.source_page, label: item.label, + category: item.category, value: item.value, scale: item.scale, concept: nil + } + end + claims + end + + def build_prompt(claims, page_map) + requests = claims.values.map do |claim| + <<~CLAIM + - id: #{claim.fetch(:id)} + kind: #{claim.fetch(:kind)} + target label: #{claim.fetch(:label)} + target category: #{claim.fetch(:category).presence || "(none)"} + excerpt page: #{page_map.fetch(claim.fetch(:physical_page))} + CLAIM + end.join + <<~PROMPT + Independently transcribe cited current-year cells from the attached Canadian municipal financial statement excerpt. + Fiscal year: #{@extraction.fiscal_year_end.year} + + For every request below, inspect only its stated excerpt page. Return the exact printed label, category (or an empty + string for facts), current-year actual numeric cell, printed column-year heading, and excerpt page. Mark found=false + if the row or exact current-year cell is absent, ambiguous, illegible, budget-only, or comparative-only. Never calculate, + infer, substitute a total, or copy a value from another column. Do not omit a request. + + REQUESTS + #{requests} + PROMPT + end + + def apply_response(checks, claims, page_map, response) + rows = response.fetch("claims") + raise ArgumentError, "visual claims must be an array" unless rows.is_a?(Array) + ids = rows.map { _1.fetch("id") } + raise ArgumentError, "visual claim ids must be unique" unless ids.uniq.length == ids.length + raise ArgumentError, "visual claim ids do not match requests" unless ids.sort == claims.keys.sort + + verdicts = rows.to_h do |row| + claim = claims.fetch(row.fetch("id")) + [ claim.fetch(:id), verify_claim(claim, page_map, row) ] + end + updated = checks.map do |existing| + verdict = verdicts[existing[:id]] + verdict&.fetch(:passed) ? check(existing[:id], "pass", verdict.fetch(:detail)) : existing + end + visual_checks = verdicts.map do |id, verdict| + check("visual_evidence:#{id}", verdict.fetch(:passed) ? "pass" : "fail", verdict.fetch(:detail)) + end + [ *updated, *visual_checks ] + end + + def verify_claim(claim, page_map, row) + expected_excerpt = page_map.fetch(claim.fetch(:physical_page)) + failures = [] + failures << "not found or ambiguous" unless row.fetch("found") + failures << "wrong excerpt page" unless Integer(row.fetch("excerpt_page")) == expected_excerpt + failures << "label mismatch" unless evidence_key(row.fetch("transcribed_label")) == evidence_key(claim.fetch(:label)) + if claim.fetch(:kind) == "line_item" + failures << "category mismatch" unless evidence_key(row.fetch("transcribed_category")) == evidence_key(claim.fetch(:category)) + end + column_year = row.fetch("column_year").to_s + current_year = @extraction.fiscal_year_end.year.to_s + failures << "wrong year or non-actual column" unless column_year.match?(/(? error + { passed: false, detail: "visual model=#{@model}; physical_page=#{claim.fetch(:physical_page)}; #{error.message}" } + end + + def call_ruby_llm(prompt:, pdf_path:) + RubyLLM.chat(model: @model) + .with_temperature(0) + .with_schema(Warehouse::FinancialStatementExtraction::VisualEvidenceResponseSchema) + .ask(prompt, with: pdf_path) + end + + def line_item_key(item) + [ item.flow, item.category, item.label ].join(":").parameterize + end + + def evidence_key(value) = value.to_s.parameterize + def check(id, status, detail) = { id:, status:, detail: } +end diff --git a/app/models/warehouse/financial_statement_line_item.rb b/app/models/warehouse/financial_statement_line_item.rb new file mode 100644 index 00000000..10986cb9 --- /dev/null +++ b/app/models/warehouse/financial_statement_line_item.rb @@ -0,0 +1,18 @@ +class Warehouse::FinancialStatementLineItem < Warehouse::Record + FLOWS = %w[revenue expense].freeze + SCALES = Warehouse::FinancialStatementFact::SCALES + + belongs_to :financial_statement_extraction, + class_name: "Warehouse::FinancialStatementExtraction", + inverse_of: :financial_statement_line_items + + validates :flow, inclusion: { in: FLOWS } + validates :category, :label, :raw_text, :column_year, presence: true + validates :scale, inclusion: { in: SCALES } + validates :source_page, numericality: { only_integer: true, greater_than: 0 } + validates :position, numericality: { only_integer: true, greater_than_or_equal_to: 0 }, + uniqueness: { scope: [ :financial_statement_extraction_id, :flow ] } + validates :extraction_confidence, + numericality: { greater_than_or_equal_to: 0, less_than_or_equal_to: 1 }, + allow_nil: true +end diff --git a/config/routes.rb b/config/routes.rb index 08afc950..7d9e6d2c 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -97,6 +97,10 @@ namespace :warehouse do resources :jurisdictions, only: [ :index ] + get "municipal_financial_statements", to: "municipal_financial_statements#index" + get "municipal_financial_statements/:province/:municipality", to: "municipal_financial_statements#show" + get "municipal_financial_statements/:province/:municipality/:year", + to: "municipal_financial_statements#show", constraints: { year: /\d{4}/ } end namespace :metrics do diff --git a/db/migrate/20260829000001_create_financial_statement_line_items.rb b/db/migrate/20260829000001_create_financial_statement_line_items.rb new file mode 100644 index 00000000..453cfa24 --- /dev/null +++ b/db/migrate/20260829000001_create_financial_statement_line_items.rb @@ -0,0 +1,33 @@ +class CreateFinancialStatementLineItems < ActiveRecord::Migration[8.1] + def change + create_table "warehouse.financial_statement_line_items" do |t| + t.references :financial_statement_extraction, null: false, + foreign_key: { to_table: "warehouse.financial_statement_extractions" }, + index: { name: "index_financial_statement_line_items_on_extraction_id" } + t.string :flow, null: false + t.string :category, null: false + t.string :label, null: false + t.decimal :value, precision: 24, scale: 2, null: false + t.string :raw_text, null: false + t.integer :scale, null: false, default: 1 + t.integer :source_page, null: false + t.string :column_year, null: false + t.integer :position, null: false + t.decimal :extraction_confidence, precision: 5, scale: 4 + t.timestamps + + t.index [ :financial_statement_extraction_id, :flow, :category, :label ], unique: true, + name: "index_financial_statement_line_items_identity" + t.index [ :financial_statement_extraction_id, :flow, :position ], + name: "index_financial_statement_line_items_order" + t.check_constraint "flow IN ('revenue','expense')", + name: "financial_statement_line_items_flow" + t.check_constraint "scale IN (1,1000,1000000)", + name: "financial_statement_line_items_scale" + t.check_constraint "source_page > 0", name: "financial_statement_line_items_source_page" + t.check_constraint "position >= 0", name: "financial_statement_line_items_position" + t.check_constraint "extraction_confidence IS NULL OR (extraction_confidence >= 0 AND extraction_confidence <= 1)", + name: "financial_statement_line_items_confidence" + end + end +end diff --git a/db/migrate/20260829000002_create_census_profiles.rb b/db/migrate/20260829000002_create_census_profiles.rb new file mode 100644 index 00000000..b7d813f1 --- /dev/null +++ b/db/migrate/20260829000002_create_census_profiles.rb @@ -0,0 +1,27 @@ +class CreateCensusProfiles < ActiveRecord::Migration[8.1] + def change + create_table "warehouse.census_profiles" do |t| + t.integer :census_year, null: false + t.string :geo_level, null: false + t.string :geo_uid, null: false + t.integer :population, null: false + t.decimal :area_sq_km + t.decimal :population_density_per_sq_km + t.string :source_url, null: false + t.string :source_sha256, null: false + t.datetime :retrieved_at, null: false + t.timestamps + + t.index [ :census_year, :geo_level, :geo_uid, :source_sha256 ], unique: true, + name: "index_census_profiles_vintage_geography_source" + t.index [ :census_year, :geo_level, :geo_uid, :retrieved_at ], + name: "index_census_profiles_latest" + t.check_constraint "census_year > 0", name: "census_profiles_year" + t.check_constraint "population > 0", name: "census_profiles_population" + t.check_constraint "area_sq_km IS NULL OR area_sq_km > 0", name: "census_profiles_area" + t.check_constraint "population_density_per_sq_km IS NULL OR population_density_per_sq_km >= 0", + name: "census_profiles_density" + t.check_constraint "source_sha256 ~ '^[0-9a-f]{64}$'", name: "census_profiles_sha256" + end + end +end diff --git a/db/migrate/20260829000003_add_year_to_financial_statement_extraction_identity.rb b/db/migrate/20260829000003_add_year_to_financial_statement_extraction_identity.rb new file mode 100644 index 00000000..dd3d61c1 --- /dev/null +++ b/db/migrate/20260829000003_add_year_to_financial_statement_extraction_identity.rb @@ -0,0 +1,9 @@ +class AddYearToFinancialStatementExtractionIdentity < ActiveRecord::Migration[8.1] + def change + remove_index "warehouse.financial_statement_extractions", + name: "index_financial_statement_extractions_source_version" + add_index "warehouse.financial_statement_extractions", + [ :institution_release_id, :asset_sha256, :extractor_version, :fiscal_year_end ], + unique: true, name: "index_financial_statement_extractions_source_version_year" + end +end diff --git a/db/migrate/20260829000004_allow_repeated_financial_statement_line_labels.rb b/db/migrate/20260829000004_allow_repeated_financial_statement_line_labels.rb new file mode 100644 index 00000000..21e6cbd0 --- /dev/null +++ b/db/migrate/20260829000004_allow_repeated_financial_statement_line_labels.rb @@ -0,0 +1,11 @@ +class AllowRepeatedFinancialStatementLineLabels < ActiveRecord::Migration[8.1] + def change + remove_index "warehouse.financial_statement_line_items", + name: "index_financial_statement_line_items_identity" + remove_index "warehouse.financial_statement_line_items", + name: "index_financial_statement_line_items_order" + add_index "warehouse.financial_statement_line_items", + [ :financial_statement_extraction_id, :flow, :position ], unique: true, + name: "index_financial_statement_line_items_order" + end +end diff --git a/db/migrate/20260829000005_require_approved_financial_statement_checks.rb b/db/migrate/20260829000005_require_approved_financial_statement_checks.rb new file mode 100644 index 00000000..a3f202fb --- /dev/null +++ b/db/migrate/20260829000005_require_approved_financial_statement_checks.rb @@ -0,0 +1,8 @@ +class RequireApprovedFinancialStatementChecks < ActiveRecord::Migration[8.0] + def change + add_check_constraint :financial_statement_extractions, + "status <> 'approved' OR (jsonb_typeof(check_results) = 'array' AND jsonb_array_length(check_results) > 0)", + name: "financial_statement_extractions_approved_checks", + schema: "warehouse" + end +end diff --git a/db/migrate/20260829000006_require_approved_financial_statement_review.rb b/db/migrate/20260829000006_require_approved_financial_statement_review.rb new file mode 100644 index 00000000..c2e9e15e --- /dev/null +++ b/db/migrate/20260829000006_require_approved_financial_statement_review.rb @@ -0,0 +1,8 @@ +class RequireApprovedFinancialStatementReview < ActiveRecord::Migration[8.0] + def change + add_check_constraint :financial_statement_extractions, + "status <> 'approved' OR (reviewed_at IS NOT NULL AND reviewed_by IS NOT NULL)", + name: "financial_statement_extractions_approved_review", + schema: "warehouse" + end +end diff --git a/db/migrate/20260829000007_require_completed_financial_statement_checks.rb b/db/migrate/20260829000007_require_completed_financial_statement_checks.rb new file mode 100644 index 00000000..a552291d --- /dev/null +++ b/db/migrate/20260829000007_require_completed_financial_statement_checks.rb @@ -0,0 +1,9 @@ +class RequireCompletedFinancialStatementChecks < ActiveRecord::Migration[8.0] + def change + add_check_constraint :financial_statement_extractions, + "status NOT IN ('extracted', 'needs_review', 'approved', 'rejected', 'failed') " \ + "OR (jsonb_typeof(check_results) = 'array' AND jsonb_array_length(check_results) > 0)", + name: "financial_statement_extractions_completed_checks", + schema: "warehouse" + end +end diff --git a/db/structure.sql b/db/structure.sql index 31556232..7736881d 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -2756,6 +2756,50 @@ CREATE SEQUENCE warehouse.canonical_observations_id_seq ALTER SEQUENCE warehouse.canonical_observations_id_seq OWNED BY warehouse.canonical_observations.id; +-- +-- Name: census_profiles; Type: TABLE; Schema: warehouse; Owner: - +-- + +CREATE TABLE warehouse.census_profiles ( + id bigint NOT NULL, + census_year integer NOT NULL, + geo_level character varying NOT NULL, + geo_uid character varying NOT NULL, + population integer NOT NULL, + area_sq_km numeric, + population_density_per_sq_km numeric, + source_url character varying NOT NULL, + source_sha256 character varying NOT NULL, + retrieved_at timestamp(6) without time zone NOT NULL, + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL, + CONSTRAINT census_profiles_area CHECK (((area_sq_km IS NULL) OR (area_sq_km > (0)::numeric))), + CONSTRAINT census_profiles_density CHECK (((population_density_per_sq_km IS NULL) OR (population_density_per_sq_km >= (0)::numeric))), + CONSTRAINT census_profiles_population CHECK ((population > 0)), + CONSTRAINT census_profiles_sha256 CHECK (((source_sha256)::text ~ '^[0-9a-f]{64}$'::text)), + CONSTRAINT census_profiles_year CHECK ((census_year > 0)) +); + + +-- +-- Name: census_profiles_id_seq; Type: SEQUENCE; Schema: warehouse; Owner: - +-- + +CREATE SEQUENCE warehouse.census_profiles_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: census_profiles_id_seq; Type: SEQUENCE OWNED BY; Schema: warehouse; Owner: - +-- + +ALTER SEQUENCE warehouse.census_profiles_id_seq OWNED BY warehouse.census_profiles.id; + + -- -- Name: composition_validation_results; Type: TABLE; Schema: warehouse; Owner: - -- @@ -3366,11 +3410,14 @@ CREATE TABLE warehouse.financial_statement_extractions ( review_notes text, created_at timestamp(6) without time zone NOT NULL, updated_at timestamp(6) without time zone NOT NULL, - CONSTRAINT financial_statement_extractions_basis CHECK (((statement_basis)::text = ANY ((ARRAY['consolidated'::character varying, 'non_consolidated'::character varying])::text[]))), - CONSTRAINT financial_statement_extractions_language CHECK (((language IS NULL) OR ((language)::text = ANY ((ARRAY['en'::character varying, 'fr'::character varying, 'bilingual'::character varying])::text[])))), + CONSTRAINT financial_statement_extractions_approved_checks CHECK ((((status)::text <> 'approved'::text) OR ((jsonb_typeof(check_results) = 'array'::text) AND (jsonb_array_length(check_results) > 0)))), + CONSTRAINT financial_statement_extractions_approved_review CHECK ((((status)::text <> 'approved'::text) OR ((reviewed_at IS NOT NULL) AND (reviewed_by IS NOT NULL)))), + CONSTRAINT financial_statement_extractions_basis CHECK (((statement_basis)::text = ANY (ARRAY[('consolidated'::character varying)::text, ('non_consolidated'::character varying)::text]))), + CONSTRAINT financial_statement_extractions_completed_checks CHECK ((((status)::text <> ALL ((ARRAY['extracted'::character varying, 'needs_review'::character varying, 'approved'::character varying, 'rejected'::character varying, 'failed'::character varying])::text[])) OR ((jsonb_typeof(check_results) = 'array'::text) AND (jsonb_array_length(check_results) > 0)))), + CONSTRAINT financial_statement_extractions_language CHECK (((language IS NULL) OR ((language)::text = ANY (ARRAY[('en'::character varying)::text, ('fr'::character varying)::text, ('bilingual'::character varying)::text])))), CONSTRAINT financial_statement_extractions_reviewer_pair CHECK (((reviewed_at IS NULL) = (reviewed_by IS NULL))), CONSTRAINT financial_statement_extractions_sha256 CHECK (((asset_sha256)::text ~ '^[0-9a-f]{64}$'::text)), - CONSTRAINT financial_statement_extractions_status CHECK (((status)::text = ANY ((ARRAY['pending'::character varying, 'extracting'::character varying, 'extracted'::character varying, 'needs_review'::character varying, 'approved'::character varying, 'rejected'::character varying, 'failed'::character varying])::text[]))) + CONSTRAINT financial_statement_extractions_status CHECK (((status)::text = ANY (ARRAY[('pending'::character varying)::text, ('extracting'::character varying)::text, ('extracted'::character varying)::text, ('needs_review'::character varying)::text, ('approved'::character varying)::text, ('rejected'::character varying)::text, ('failed'::character varying)::text]))) ); @@ -3411,11 +3458,11 @@ CREATE TABLE warehouse.financial_statement_facts ( extraction_confidence numeric(5,4), created_at timestamp(6) without time zone NOT NULL, updated_at timestamp(6) without time zone NOT NULL, - CONSTRAINT financial_statement_facts_concept CHECK (((concept)::text = ANY ((ARRAY['total_financial_assets'::character varying, 'total_liabilities'::character varying, 'net_financial_assets'::character varying, 'total_non_financial_assets'::character varying, 'accumulated_surplus'::character varying, 'opening_accumulated_surplus'::character varying, 'total_revenue'::character varying, 'total_expenses'::character varying, 'annual_surplus'::character varying])::text[]))), + CONSTRAINT financial_statement_facts_concept CHECK (((concept)::text = ANY (ARRAY[('total_financial_assets'::character varying)::text, ('total_liabilities'::character varying)::text, ('net_financial_assets'::character varying)::text, ('total_non_financial_assets'::character varying)::text, ('accumulated_surplus'::character varying)::text, ('opening_accumulated_surplus'::character varying)::text, ('total_revenue'::character varying)::text, ('total_expenses'::character varying)::text, ('annual_surplus'::character varying)::text]))), CONSTRAINT financial_statement_facts_confidence CHECK (((extraction_confidence IS NULL) OR ((extraction_confidence >= (0)::numeric) AND (extraction_confidence <= (1)::numeric)))), CONSTRAINT financial_statement_facts_scale CHECK ((scale = ANY (ARRAY[1, 1000, 1000000]))), CONSTRAINT financial_statement_facts_source_page CHECK ((source_page > 0)), - CONSTRAINT financial_statement_facts_statement CHECK (((statement)::text = ANY ((ARRAY['financial_position'::character varying, 'operations'::character varying, 'accumulated_surplus'::character varying])::text[]))) + CONSTRAINT financial_statement_facts_statement CHECK (((statement)::text = ANY (ARRAY[('financial_position'::character varying)::text, ('operations'::character varying)::text, ('accumulated_surplus'::character varying)::text]))) ); @@ -3438,6 +3485,52 @@ CREATE SEQUENCE warehouse.financial_statement_facts_id_seq ALTER SEQUENCE warehouse.financial_statement_facts_id_seq OWNED BY warehouse.financial_statement_facts.id; +-- +-- Name: financial_statement_line_items; Type: TABLE; Schema: warehouse; Owner: - +-- + +CREATE TABLE warehouse.financial_statement_line_items ( + id bigint NOT NULL, + financial_statement_extraction_id bigint NOT NULL, + flow character varying NOT NULL, + category character varying NOT NULL, + label character varying NOT NULL, + value numeric(24,2) NOT NULL, + raw_text character varying NOT NULL, + scale integer DEFAULT 1 NOT NULL, + source_page integer NOT NULL, + column_year character varying NOT NULL, + "position" integer NOT NULL, + extraction_confidence numeric(5,4), + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL, + CONSTRAINT financial_statement_line_items_confidence CHECK (((extraction_confidence IS NULL) OR ((extraction_confidence >= (0)::numeric) AND (extraction_confidence <= (1)::numeric)))), + CONSTRAINT financial_statement_line_items_flow CHECK (((flow)::text = ANY ((ARRAY['revenue'::character varying, 'expense'::character varying])::text[]))), + CONSTRAINT financial_statement_line_items_position CHECK (("position" >= 0)), + CONSTRAINT financial_statement_line_items_scale CHECK ((scale = ANY (ARRAY[1, 1000, 1000000]))), + CONSTRAINT financial_statement_line_items_source_page CHECK ((source_page > 0)) +); + + +-- +-- Name: financial_statement_line_items_id_seq; Type: SEQUENCE; Schema: warehouse; Owner: - +-- + +CREATE SEQUENCE warehouse.financial_statement_line_items_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: financial_statement_line_items_id_seq; Type: SEQUENCE OWNED BY; Schema: warehouse; Owner: - +-- + +ALTER SEQUENCE warehouse.financial_statement_line_items_id_seq OWNED BY warehouse.financial_statement_line_items.id; + + -- -- Name: fiscal_authorities; Type: TABLE; Schema: warehouse; Owner: - -- @@ -3734,7 +3827,7 @@ CREATE TABLE warehouse.institution_coverages ( created_at timestamp(6) without time zone NOT NULL, updated_at timestamp(6) without time zone NOT NULL, CONSTRAINT institution_coverages_status CHECK (((status)::text = ANY (ARRAY[('complete'::character varying)::text, ('partial'::character varying)::text, ('not-searched'::character varying)::text, ('not-found'::character varying)::text, ('unavailable'::character varying)::text, ('failed'::character varying)::text]))), - CONSTRAINT institution_coverages_subject CHECK (((subject)::text = ANY ((ARRAY['institutions'::character varying, 'websites'::character varying, 'geographies'::character varying, 'relationships'::character varying, 'financial-statements'::character varying, 'annual-reports'::character varying, 'statement-of-financial-information'::character varying, 'financial-data-return'::character varying, 'document-assets'::character varying, 'csd-inventory'::character varying, 'csd-authority-mapping'::character varying])::text[]))) + CONSTRAINT institution_coverages_subject CHECK (((subject)::text = ANY (ARRAY[('institutions'::character varying)::text, ('websites'::character varying)::text, ('geographies'::character varying)::text, ('relationships'::character varying)::text, ('financial-statements'::character varying)::text, ('annual-reports'::character varying)::text, ('statement-of-financial-information'::character varying)::text, ('financial-data-return'::character varying)::text, ('document-assets'::character varying)::text, ('csd-inventory'::character varying)::text, ('csd-authority-mapping'::character varying)::text]))) ); @@ -3871,8 +3964,8 @@ CREATE TABLE warehouse.institution_geographies ( valid_to date, notes text, CONSTRAINT institution_geographies_confidence CHECK (((confidence IS NULL) OR ((confidence >= (0)::numeric) AND (confidence <= (1)::numeric)))), - CONSTRAINT institution_geographies_match_method CHECK (((match_method)::text = ANY ((ARRAY['legacy'::character varying, 'authoritative_crosswalk'::character varying, 'source_assertion'::character varying, 'exact_identifier'::character varying, 'exact_name'::character varying, 'jurisdictional_fallback'::character varying])::text[]))), - CONSTRAINT institution_geographies_role_v2 CHECK (((role)::text = ANY ((ARRAY['governs'::character varying, 'administers'::character varying, 'serves'::character varying, 'headquartered_in'::character varying])::text[]))), + CONSTRAINT institution_geographies_match_method CHECK (((match_method)::text = ANY (ARRAY[('legacy'::character varying)::text, ('authoritative_crosswalk'::character varying)::text, ('source_assertion'::character varying)::text, ('exact_identifier'::character varying)::text, ('exact_name'::character varying)::text, ('jurisdictional_fallback'::character varying)::text]))), + CONSTRAINT institution_geographies_role_v2 CHECK (((role)::text = ANY (ARRAY[('governs'::character varying)::text, ('administers'::character varying)::text, ('serves'::character varying)::text, ('headquartered_in'::character varying)::text]))), CONSTRAINT institution_geographies_valid_range CHECK (((valid_to IS NULL) OR (valid_from IS NULL) OR (valid_to >= valid_from))) ); @@ -3918,7 +4011,7 @@ CREATE TABLE warehouse.institution_geography_snapshots ( updated_at timestamp(6) without time zone NOT NULL, classification_type character varying, authority_status character varying DEFAULT 'legacy'::character varying NOT NULL, - CONSTRAINT institution_geo_snapshots_authority_status CHECK (((authority_status)::text = ANY ((ARRAY['legacy'::character varying, 'not_applicable'::character varying, 'verified'::character varying, 'provisional'::character varying, 'unresolved'::character varying])::text[]))), + CONSTRAINT institution_geo_snapshots_authority_status CHECK (((authority_status)::text = ANY (ARRAY[('legacy'::character varying)::text, ('not_applicable'::character varying)::text, ('verified'::character varying)::text, ('provisional'::character varying)::text, ('unresolved'::character varying)::text]))), CONSTRAINT institution_geo_snapshots_canonical_id_format CHECK ((((canonical_id)::text = lower((canonical_id)::text)) AND ((canonical_id)::text ~ '^ca/geography/[a-z0-9]+(-[a-z0-9]+)*/[a-z0-9]+(-[a-z0-9]+)*$'::text))) ); @@ -5724,6 +5817,13 @@ ALTER TABLE ONLY warehouse.api_tokens ALTER COLUMN id SET DEFAULT nextval('wareh ALTER TABLE ONLY warehouse.canonical_observations ALTER COLUMN id SET DEFAULT nextval('warehouse.canonical_observations_id_seq'::regclass); +-- +-- Name: census_profiles id; Type: DEFAULT; Schema: warehouse; Owner: - +-- + +ALTER TABLE ONLY warehouse.census_profiles ALTER COLUMN id SET DEFAULT nextval('warehouse.census_profiles_id_seq'::regclass); + + -- -- Name: composition_validation_results id; Type: DEFAULT; Schema: warehouse; Owner: - -- @@ -5794,6 +5894,13 @@ ALTER TABLE ONLY warehouse.financial_statement_extractions ALTER COLUMN id SET D ALTER TABLE ONLY warehouse.financial_statement_facts ALTER COLUMN id SET DEFAULT nextval('warehouse.financial_statement_facts_id_seq'::regclass); +-- +-- Name: financial_statement_line_items id; Type: DEFAULT; Schema: warehouse; Owner: - +-- + +ALTER TABLE ONLY warehouse.financial_statement_line_items ALTER COLUMN id SET DEFAULT nextval('warehouse.financial_statement_line_items_id_seq'::regclass); + + -- -- Name: fiscal_authorities id; Type: DEFAULT; Schema: warehouse; Owner: - -- @@ -6623,6 +6730,14 @@ ALTER TABLE ONLY warehouse.canonical_observations ADD CONSTRAINT canonical_observations_pkey PRIMARY KEY (id); +-- +-- Name: census_profiles census_profiles_pkey; Type: CONSTRAINT; Schema: warehouse; Owner: - +-- + +ALTER TABLE ONLY warehouse.census_profiles + ADD CONSTRAINT census_profiles_pkey PRIMARY KEY (id); + + -- -- Name: composition_validation_results composition_validation_results_pkey; Type: CONSTRAINT; Schema: warehouse; Owner: - -- @@ -6695,6 +6810,14 @@ ALTER TABLE ONLY warehouse.financial_statement_facts ADD CONSTRAINT financial_statement_facts_pkey PRIMARY KEY (id); +-- +-- Name: financial_statement_line_items financial_statement_line_items_pkey; Type: CONSTRAINT; Schema: warehouse; Owner: - +-- + +ALTER TABLE ONLY warehouse.financial_statement_line_items + ADD CONSTRAINT financial_statement_line_items_pkey PRIMARY KEY (id); + + -- -- Name: fiscal_authorities fiscal_authorities_pkey; Type: CONSTRAINT; Schema: warehouse; Owner: - -- @@ -9076,6 +9199,20 @@ CREATE UNIQUE INDEX index_api_tokens_on_name ON warehouse.api_tokens USING btree CREATE UNIQUE INDEX index_api_tokens_on_token_hash ON warehouse.api_tokens USING btree (token_hash); +-- +-- Name: index_census_profiles_latest; Type: INDEX; Schema: warehouse; Owner: - +-- + +CREATE INDEX index_census_profiles_latest ON warehouse.census_profiles USING btree (census_year, geo_level, geo_uid, retrieved_at); + + +-- +-- Name: index_census_profiles_vintage_geography_source; Type: INDEX; Schema: warehouse; Owner: - +-- + +CREATE UNIQUE INDEX index_census_profiles_vintage_geography_source ON warehouse.census_profiles USING btree (census_year, geo_level, geo_uid, source_sha256); + + -- -- Name: index_financial_statement_extractions_institution_year; Type: INDEX; Schema: warehouse; Owner: - -- @@ -9091,10 +9228,10 @@ CREATE INDEX index_financial_statement_extractions_on_release_id ON warehouse.fi -- --- Name: index_financial_statement_extractions_source_version; Type: INDEX; Schema: warehouse; Owner: - +-- Name: index_financial_statement_extractions_source_version_year; Type: INDEX; Schema: warehouse; Owner: - -- -CREATE UNIQUE INDEX index_financial_statement_extractions_source_version ON warehouse.financial_statement_extractions USING btree (institution_release_id, asset_sha256, extractor_version); +CREATE UNIQUE INDEX index_financial_statement_extractions_source_version_year ON warehouse.financial_statement_extractions USING btree (institution_release_id, asset_sha256, extractor_version, fiscal_year_end); -- @@ -9111,6 +9248,20 @@ CREATE UNIQUE INDEX index_financial_statement_facts_extraction_concept ON wareho CREATE INDEX index_financial_statement_facts_on_extraction_id ON warehouse.financial_statement_facts USING btree (financial_statement_extraction_id); +-- +-- Name: index_financial_statement_line_items_on_extraction_id; Type: INDEX; Schema: warehouse; Owner: - +-- + +CREATE INDEX index_financial_statement_line_items_on_extraction_id ON warehouse.financial_statement_line_items USING btree (financial_statement_extraction_id); + + +-- +-- Name: index_financial_statement_line_items_order; Type: INDEX; Schema: warehouse; Owner: - +-- + +CREATE UNIQUE INDEX index_financial_statement_line_items_order ON warehouse.financial_statement_line_items USING btree (financial_statement_extraction_id, flow, "position"); + + -- -- Name: index_fiscal_authorities_on_lineage_entry_id; Type: INDEX; Schema: warehouse; Owner: - -- @@ -10429,6 +10580,14 @@ ALTER TABLE ONLY warehouse.institution_geographies ADD CONSTRAINT fk_rails_2cd3747963 FOREIGN KEY (institution_release_id) REFERENCES warehouse.institution_releases(id); +-- +-- Name: financial_statement_extractions fk_rails_30abaad519; Type: FK CONSTRAINT; Schema: warehouse; Owner: - +-- + +ALTER TABLE ONLY warehouse.financial_statement_extractions + ADD CONSTRAINT fk_rails_30abaad519 FOREIGN KEY (institution_release_id) REFERENCES warehouse.institution_releases(id); + + -- -- Name: fiscal_expenditures fk_rails_34d506f249; Type: FK CONSTRAINT; Schema: warehouse; Owner: - -- @@ -10501,6 +10660,14 @@ ALTER TABLE ONLY warehouse.media_feed_fetches ADD CONSTRAINT fk_rails_75522ea188 FOREIGN KEY (media_feed_id) REFERENCES warehouse.media_feeds(id); +-- +-- Name: financial_statement_line_items fk_rails_790baef66c; Type: FK CONSTRAINT; Schema: warehouse; Owner: - +-- + +ALTER TABLE ONLY warehouse.financial_statement_line_items + ADD CONSTRAINT fk_rails_790baef66c FOREIGN KEY (financial_statement_extraction_id) REFERENCES warehouse.financial_statement_extractions(id); + + -- -- Name: fiscal_authorities fk_rails_847d2d9f3d; Type: FK CONSTRAINT; Schema: warehouse; Owner: - -- @@ -10525,14 +10692,6 @@ ALTER TABLE ONLY warehouse.spending_awards ADD CONSTRAINT fk_rails_90e55d982c FOREIGN KEY (raw_ingestion_id) REFERENCES warehouse.raw_ingestions(id); --- --- Name: financial_statement_extractions fk_rails_30abaad519; Type: FK CONSTRAINT; Schema: warehouse; Owner: - --- - -ALTER TABLE ONLY warehouse.financial_statement_extractions - ADD CONSTRAINT fk_rails_30abaad519 FOREIGN KEY (institution_release_id) REFERENCES warehouse.institution_releases(id); - - -- -- Name: financial_statement_facts fk_rails_99ce4dda9e; Type: FK CONSTRAINT; Schema: warehouse; Owner: - -- @@ -11044,6 +11203,13 @@ ALTER TABLE ONLY warehouse.source_footnotes SET search_path TO public,warehouse; INSERT INTO "schema_migrations" (version) VALUES +('20260829000007'), +('20260829000006'), +('20260829000005'), +('20260829000004'), +('20260829000003'), +('20260829000002'), +('20260829000001'), ('20260827000001'), ('20260822000001'), ('20260821000001'), diff --git a/docs/plans/municipal_budget_acquisition.md b/docs/plans/municipal_budget_acquisition.md new file mode 100644 index 00000000..a28d2a44 --- /dev/null +++ b/docs/plans/municipal_budget_acquisition.md @@ -0,0 +1,91 @@ +# Municipal budget acquisition contract + +This is a discovery and archival contract for a future public-institution release. It must not +write to the pinned `2026-08-27` release or share extraction rows with audited financial +statements. + +## Document identity + +- Document type: `budget` +- Canonical ID: `ca/{province}/{municipality}/documents/budgets/{fiscal_year}/{variant}` +- `fiscal_year` is the year being budgeted for, not the publication or retrieval year. +- Allowed variants: `operating`, `capital`, and `consolidated`. +- A consolidated budget is preferred when it contains both operating and capital plans. Separate + operating and capital documents may coexist for the same municipality-year. + +Budgets and audited financial statements for the same municipality-year are separate publishable +works. Budget records must never enter the financial-statements API arrays, audited-statement +Sankey, or audited institution-year publication slots. + +## Region ownership + +- West: `bc`, `ab`, `sk`, `mb` +- Central: `on`, `qc` +- Atlantic and territories: `nb`, `ns`, `pe`, `nl`, `yt`, `nt`, `nu` + +Each worker owns only its region's batch and log. Cross-listed or shared documents may resolve to +the same content-addressed asset without coordinating manifest writes. + +## JSONL record + +Every terminal discovery candidate, including failures, is one JSON object with these fields: + +```json +{ + "institution_canonical_id": "ca/on/example", + "institution_name": "Example", + "province": "on", + "fiscal_year": 2026, + "document_type": "budget", + "document_variant": "consolidated", + "canonical_id": "ca/on/example/documents/budgets/2026/consolidated", + "title": "2026 Approved Budget", + "language": "en", + "source_page_url": "https://example.ca/budget", + "download_url": "https://example.ca/2026-budget.pdf", + "retrieved_at": "2026-08-30T00:00:00Z", + "content_sha256": "64 lowercase hexadecimal characters", + "archive_path": "sha256/ab/abcdef...pdf", + "mime_type": "application/pdf", + "byte_size": 123456, + "status": "archived", + "checks": [ + {"id": "official_source", "status": "pass", "detail": "municipal website"}, + {"id": "pdf_signature", "status": "pass", "detail": "%PDF-"}, + {"id": "sha256", "status": "pass", "detail": "matches archived bytes"}, + {"id": "issuer", "status": "pass", "detail": "title/source identifies ca/on/example"}, + {"id": "fiscal_year", "status": "pass", "detail": "budgeted year is 2026"}, + {"id": "budget_variant", "status": "pass", "detail": "operating and capital sections"} + ] +} +``` + +Failed candidates retain the URLs and evidence available, set `status` to `failed`, leave +unavailable asset fields null, and save at least one failing check with a concrete reason. A record +is not terminal unless `checks` is a non-empty array. + +## Source and archival rules + +1. Use official municipal or official provincial repositories as the final source. Search engines + may discover a page but are not evidence. +2. Collect all available historical approved/adopted budgets. Draft consultation material and + budget highlights are not substitutes for full budgets. +3. Discovery is network-bound only while financial-statement extraction is active: no OCR, PDF + rendering, model extraction, or database connections. +4. Rate-limit requests per host and reuse the existing municipal scraper conventions. +5. Verify disk headroom before downloads. +6. Write assets atomically: download to a temporary file, verify `%PDF-`, size, and SHA-256, then + rename into the shared asset store at + `/Volumes/floppy/york_factory/public_institutions/assets/sha256/{prefix}/{sha256}.pdf`. + Treat an already-valid destination as archived and never overwrite it in place. +7. Write JSONL through a temporary region-owned file and atomically promote checkpoints. Never + allow a partial JSON line. + +## Agent prohibitions and checkpoint + +Discovery workers must not connect to the database, edit repository files, run git commands, +operate tmux, alter financial-statement processes, or write outside their own regional batch/log +and the shared content-addressed asset store. + +Stop after the first 10 terminal candidates in each region for schema and source-quality review. +Resume the full regional crawl only after that checkpoint passes. diff --git a/docs/plans/municipal_financial_statements_deployment.md b/docs/plans/municipal_financial_statements_deployment.md new file mode 100644 index 00000000..cffd2a8f --- /dev/null +++ b/docs/plans/municipal_financial_statements_deployment.md @@ -0,0 +1,124 @@ +# Municipal financial statements deployment + +This release is split into code and immutable public data. The York Factory and +CanadaSpends pull requests may be reviewed while extraction continues, but +production data must not be promoted until the national finalizer has frozen and +validated one release. + +## Pull request shape + +- York Factory is a stacked pull request based on `feat/public-institution-ontology` + (PR #111). It owns the Warehouse schema, extraction/review pipeline, public API, + census context, verification results, and release tooling. +- CanadaSpends is a stacked pull request based on + `federal-public-accounts-pipeline` (PR #275). It owns the municipal directory, + year pages, context, verification display, and Sankey rendering. +- Merge each parent before its stacked municipal pull request. Deploy York Factory + before CanadaSpends. + +## Release gates + +Use release `2026-08-27` for the current national run. Before exporting it: + +1. Let every jurisdiction handoff and reviewer finish, then run + `script/run_national_financial_finalization.zsh`. +2. Require zero detailed rows in `pending`, `extracting`, or `extracted`; zero + terminal extraction rows without saved checks; and zero approved rows without + deterministic review provenance. +3. Preserve the final coverage, numeric, scale, lineage, issuer, and test-result + artifacts alongside the release. Failures and unavailable documents remain + explicit; they are not silently dropped. +4. Run the York Factory test suite and the CanadaSpends Vitest suite/build from + the exact commits being deployed. +5. Export the immutable ontology release only after its `published_at` includes + the final review timestamps. Never edit an already-published release in place; + if the release has already been published, create a new dated release. + +## Data promotion contract + +Do not copy the local development database into production. The promotion unit is +one immutable, checksummed release directory produced by: + +```sh +bin/rails institution_ontology:export[2026-08-27,tmp/public-institutions-2026-08-27] +``` + +The exported directory must contain the ontology, documents, document-asset +metadata, approved extractions, facts, detailed revenue/expense line items, +census context, manifest, SQL loader, and `SHA256SUMS`. Upload it under a +versioned R2 key such as +`municipal-financial-statements/releases/2026-08-27/`; never overwrite that key. +Archived source binaries remain content-addressed by SHA-256 in the archival R2 +bucket. + +The API reads the Rails `warehouse` schema, while the exporter's generic SQL +loader targets the read-only `public_institutions` interchange schema. Therefore +the release is not production-ready until York Factory has a verified, +idempotent importer that maps the release's natural identifiers back into the +existing Warehouse release. That importer must: + +- verify `SHA256SUMS`, `manifest.json`, release version, schema version, and every + referenced document/asset before opening a transaction; +- refuse a different payload for an already-imported release; +- upsert only census profiles and the selected release's approved extractions, + facts, and line items by natural identity, without using local database IDs; +- save the import manifest SHA-256 and row counts for audit and retry safety; +- stage and validate all rows, then make them visible atomically; +- provide a dry-run that performs every validation without writing. + +Until that importer is implemented and tested, the supported fallback is to run +the deterministic extraction/review pipeline against the imported immutable +source release in production. A whole-database dump/restore is not a supported +promotion path. + +## York Factory deployment + +After PR #111 and the stacked York Factory PR merge: + +```sh +bin/kamal deploy +bin/kamal app exec --reuse 'bin/rails db:migrate:status' +``` + +The seven municipal migrations are additive except for replacing extraction and +line-item unique indexes and adding check constraints. Before deploy, verify the +production table has no rows that violate the new constraints. Keep the API +unreferenced by CanadaSpends until migrations and data promotion succeed. + +Promote data only through the verified importer described above (first with +`DRY_RUN=1`), then verify at minimum: + +```text +GET /api/v1/warehouse/municipal_financial_statements +GET /api/v1/warehouse/municipal_financial_statements/on/toronto/2025 +GET /api/v1/warehouse/municipal_financial_statements/ns/halifax +``` + +Each published statement must expose a non-empty verification check list. Sample +records must also have source links, census context where mapped, and balanced +Sankey data where detailed line items passed validation. + +## CanadaSpends deployment + +Set the production server-side environment variable: + +```text +YORK_FACTORY_API_URL=https://yorkfactory.buildcanada.com/api/v1 +``` + +Merge and deploy the stacked CanadaSpends pull request only after the York API +smoke tests pass. Verify the municipal directory, Toronto's newest reviewed year, +one Atlantic municipality, bilingual routes, year switching, source links, +verification results, per-capita context, and the Sankey. + +The Cloudflare quick tunnel is a disposable preview only. It is not a production +origin and its URL must not be committed or configured in production. + +## Rollback + +- Roll CanadaSpends back first; York's new API can remain unused. +- Roll York code back only while leaving additive tables/columns in place. Do not + reverse schema migrations during an incident. +- If a data payload is wrong, disable the frontend route or mark the affected + release unpublished, retain the import audit, and promote a corrected new + release. Do not mutate the immutable R2 key or erase failed test evidence. diff --git a/docs/plans/municipal_zero_publication_remediation.md b/docs/plans/municipal_zero_publication_remediation.md new file mode 100644 index 00000000..7973ded4 --- /dev/null +++ b/docs/plans/municipal_zero_publication_remediation.md @@ -0,0 +1,52 @@ +# Municipal zero-publication remediation + +This follow-up starts only after `script/run_national_financial_finalization.zsh` exits successfully +and its final coverage, test, build, local, and tunnel artifacts exist. It must not overlap the +frozen finalizer or write to any of its output paths. + +## 1. Establish the post-finalization gap + +- Recompute collected municipalities and approved detailed years from release `2026-08-27`. +- Resolve shared PDF slots by `[asset_sha256, fiscal_year_end]` using the existing variant priority + and document-id ownership rules. Do not use a naive candidate-minus-approved count. +- Save a refuse-if-present `gap_census-.json` containing every gap municipality and all of + its candidate documents, years, variants, hashes, and headline/detailed extraction states. + +## 2. Reuse the existing extraction machinery + +- For every municipality without an approved detailed year, select the newest candidate, then the + existing variant priority, then the lowest document id. +- Run bounded `process_municipal_financial_statements.rb --document-ids` lanes against a new output + root and run a finite `review_extracted_municipal_financial_statements.rb --batch-size N` pass. +- Reuse the shared OCR cache. Do not add pipeline classes, prompts, reviewer identities, or watch + sessions. +- Iterate the next-newest candidate year only while the municipality remains unpublished. Stop + when one detailed year is approved or every candidate year is terminal with saved checks. + +## 3. Derive exhaustion and gate York + +- Save `exhausted_evidence-.json`; exhaustion is derived only when every candidate year has + a terminal detailed extraction with checks, error evidence, and review provenance. Do not add an + `exhausted` database state. +- Assert zero active headline/detailed rows, shared-asset-aware reconciliation, deterministic + reviewers for approvals, and that every gap-census municipality is approved or mechanically + exhausted. +- Write a fresh coverage audit and run the same York tests, RuboCop targets, and Zeitwerk check as + the national finalizer. + +## 4. Represent every collected municipality safely + +- Add a read-only York endpoint, or an explicit `include=collected` index mode, returning every + collected municipality with its published years and `published` or derived `exhausted` status. +- Never return financial facts or line items from unapproved extractions. +- Make CanadaSpends list all collected municipalities. Published entries route only to approved + years; exhausted entries route to a source/status page based on the existing First Nations + no-data pattern and render no financial numbers. +- Add a runtime approval guard and pure-function tests for routing/status behavior. Treat + `pnpm test` and `pnpm build` as smoke checks, not sufficient completion evidence. + +## 5. Verify the actual user surface + +Test locally and through a fresh Cloudflare tunnel against several municipalities newly approved by +the remediation pass, at least one exhausted municipality with zero numeric leakage, and Toronto as +a regression case. Save the API responses, rendered pages, redirects, and test logs with the run. diff --git a/docs/reviews/national_municipal_release_2026-08-27.md b/docs/reviews/national_municipal_release_2026-08-27.md index def49a5c..2a976f2c 100644 --- a/docs/reviews/national_municipal_release_2026-08-27.md +++ b/docs/reviews/national_municipal_release_2026-08-27.md @@ -27,6 +27,8 @@ All 13 manifests have zero validation errors and the coverage audits have zero a The validator emits 17 non-blocking duplicate-content warnings: 3 in Alberta and 14 in New Brunswick. They represent predecessor/successor duplication or an explicitly shared regional report. The PAAC and Le Goulet/Shippagan misattributions identified during review were corrected before v6. Remaining warnings stay explicit for human review. +For financial extraction coverage, 9 of the 14 New Brunswick warnings are duplicate preferred financial-statement identities. The other 5 hashes (`23d4035a…`, `27a47f8e…`, `7ab8acd2…`, `7c72ca25…`, and `d3d007e2…`) occur only on annual-report documents and are outside the preferred financial-statement candidate set. Extraction coverage reports the 9 non-owner candidates as `shared_asset` rather than duplicating extraction rows or treating them as unattempted. + ## Deliberate limitations - Ten-year completion means any ten distinct fiscal years; it does not require a contiguous or recent ten-year window. diff --git a/script/apply_prairie_parser_upgrade_audit.rb b/script/apply_prairie_parser_upgrade_audit.rb new file mode 100644 index 00000000..87b2bfdd --- /dev/null +++ b/script/apply_prairie_parser_upgrade_audit.rb @@ -0,0 +1,59 @@ +#!/usr/bin/env ruby + +require_relative "../config/environment" +require "optparse" + +options = {} +OptionParser.new do |parser| + parser.on("--audit PATH") { options[:audit] = Pathname(_1).expand_path } + parser.on("--apply") { options[:apply] = true } +end.parse! +abort "missing option: --audit" unless options[:audit] +abort "missing audit #{options[:audit]}" unless options[:audit].file? + +rows = options[:audit].each_line.filter_map do |line| + row = JSON.parse(line) + row if row["extraction_id"] && row["status"].in?(%w[pass mismatch failed]) +rescue JSON::ParserError + nil +end + +counts = Hash.new(0) +rows.each do |row| + extraction = Warehouse::FinancialStatementExtraction.find(row.fetch("extraction_id")) + from_parser = row.fetch("from_parser") + to_parser = row.fetch("to_parser") + unless extraction.llm_response_snapshot&.fetch("parser", nil) == from_parser + raise "extraction #{extraction.id} parser changed after audit" + end + unless extraction.status == "approved" + raise "extraction #{extraction.id} is no longer approved" + end + + passed = row.fetch("status") == "pass" + check = { + id: "parser_upgrade_reparse", + status: passed ? "pass" : "fail", + detail: if passed + "#{from_parser} values reproduced exactly by #{to_parser} without arithmetic fallback" + else + "#{to_parser} could not reproduce the prior arithmetic-fallback result; see saved upgrade audit" + end + } + attributes = { + check_results: Array(extraction.check_results).reject do |existing| + existing.stringify_keys["id"] == check.fetch(:id) + end + [ check ] + } + unless passed + attributes[:status] = "rejected" + attributes[:review_notes] = [ extraction.review_notes, + "Removed from publication after #{to_parser} fallback-risk audit" ].compact.join("; ") + end + extraction.update!(attributes) if options[:apply] + counts[passed ? "verified" : "rejected"] += 1 + puts({ extraction_id: extraction.id, document: extraction.document_canonical_id, + action: passed ? "verified" : "rejected", applied: options[:apply] || false }.to_json) +end + +puts({ audit: options[:audit].to_s, applied: options[:apply] || false, summary: counts }.to_json) diff --git a/script/audit_financial_statement_numeric_values.rb b/script/audit_financial_statement_numeric_values.rb new file mode 100644 index 00000000..6ac5b1b7 --- /dev/null +++ b/script/audit_financial_statement_numeric_values.rb @@ -0,0 +1,50 @@ +#!/usr/bin/env ruby + +require_relative "../config/environment" +require "optparse" + +options = { release: "2026-08-27", limit: nil } +OptionParser.new do |parser| + parser.on("--release VERSION") { options[:release] = _1 } + parser.on("--limit COUNT", Integer) { options[:limit] = _1 } +end.parse! + +release = Warehouse::InstitutionRelease.find_by!(version: options.fetch(:release)) +scope = release.financial_statement_extractions.where( + extractor_version: Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION, + status: %w[extracted needs_review approved rejected] +).order(:id) +scope = scope.limit(options[:limit]) if options[:limit] +counts = Hash.new(0) + +scope.find_each do |extraction| + mismatches = [] + extraction.financial_statement_facts.find_each do |fact| + expected = Warehouse::FinancialStatementExtraction::NumberParser.parse( + fact.raw_text, raw_label: fact.raw_label, concept: fact.concept + ) * fact.scale + next if expected == fact.value + + mismatches << { type: "fact", id: fact.id, raw_text: fact.raw_text, + stored: fact.value.to_s, expected: expected.to_s } + end + extraction.financial_statement_line_items.find_each do |item| + expected = Warehouse::FinancialStatementExtraction::NumberParser.parse(item.raw_text) * item.scale + next if expected == item.value + + mismatches << { type: "line_item", id: item.id, raw_text: item.raw_text, + stored: item.value.to_s, expected: expected.to_s } + end + result = mismatches.empty? ? "pass" : "mismatch" + counts[result] += 1 + puts({ id: extraction.id, institution: extraction.institution_canonical_id, + fiscal_year: extraction.fiscal_year_end.year, status: extraction.status, + result:, mismatches: }.to_json) +rescue => error + counts["error"] += 1 + puts({ id: extraction.id, institution: extraction.institution_canonical_id, + fiscal_year: extraction.fiscal_year_end.year, status: extraction.status, + result: "error", error: "#{error.class}: #{error.message}" }.to_json) +end + +puts({ summary: counts }.to_json) diff --git a/script/audit_financial_statement_scales.rb b/script/audit_financial_statement_scales.rb new file mode 100755 index 00000000..e88dec09 --- /dev/null +++ b/script/audit_financial_statement_scales.rb @@ -0,0 +1,57 @@ +#!/usr/bin/env ruby + +require_relative "../config/environment" +require "optparse" + +options = { release: "2026-08-27", limit: nil, asset_root: Warehouse::FinancialStatementExtraction::CandidateSet::DEFAULT_ASSET_ROOT } +OptionParser.new do |parser| + parser.on("--release VERSION") { options[:release] = _1 } + parser.on("--limit COUNT", Integer) { options[:limit] = _1 } + parser.on("--shard INDEX/COUNT") do |value| + options[:shard_index], options[:shard_count] = value.split("/", 2).map { |part| Integer(part) } + end + parser.on("--asset-root PATH") { options[:asset_root] = Pathname(_1).expand_path } +end.parse! + +release = Warehouse::InstitutionRelease.find_by!(version: options[:release]) +scope = release.financial_statement_extractions.where( + extractor_version: Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION, + status: %w[extracted approved] +).where("jsonb_exists(llm_response_snapshot, 'parser')").order(:id) +scope = scope.where("MOD(id, ?) = ?", options[:shard_count], options[:shard_index]) if options[:shard_count] +scope = scope.limit(options[:limit]) if options[:limit] +asset_root = Pathname(options[:asset_root]).expand_path +counts = Hash.new(0) + +scope.find_each do |extraction| + document = release.institution_documents.find_by!(canonical_id: extraction.document_canonical_id) + asset = document.institution_document_assets.find_by!(content_sha256: extraction.asset_sha256) + pdf_path = asset_root.join(asset.archive_path).expand_path + raise "asset path escapes root" unless pdf_path.to_s.start_with?("#{asset_root}/") + raise "missing archived PDF" unless pdf_path.file? + + pages = (extraction.financial_statement_facts.pluck(:source_page) + + extraction.financial_statement_line_items.pluck(:source_page)).uniq.sort + texts = pages.map do |page| + stdout, stderr, status = Open3.capture3( + "pdftotext", "-f", page.to_s, "-l", page.to_s, "-layout", "-enc", "UTF-8", pdf_path.to_s, "-" + ) + raise "pdftotext page #{page} failed: #{stderr}" unless status.success? + stdout.force_encoding(Encoding::UTF_8).scrub + end + source_scale = Warehouse::FinancialStatementExtraction::ScaleDetector.detect(texts) + stored_scales = (extraction.financial_statement_facts.pluck(:scale) + + extraction.financial_statement_line_items.pluck(:scale)).uniq.sort + result = stored_scales == [ source_scale ] ? "pass" : "mismatch" + counts[result] += 1 + puts({ id: extraction.id, institution: extraction.institution_canonical_id, + fiscal_year: extraction.fiscal_year_end.year, status: extraction.status, + result:, source_scale:, stored_scales: }.to_json) +rescue => error + counts["error"] += 1 + puts({ id: extraction.id, institution: extraction.institution_canonical_id, + fiscal_year: extraction.fiscal_year_end.year, result: "error", + error: "#{error.class}: #{error.message}" }.to_json) +end + +puts({ summary: counts }.to_json) diff --git a/script/audit_municipal_financial_extraction_coverage.rb b/script/audit_municipal_financial_extraction_coverage.rb new file mode 100644 index 00000000..4134ff06 --- /dev/null +++ b/script/audit_municipal_financial_extraction_coverage.rb @@ -0,0 +1,23 @@ +#!/usr/bin/env ruby + +require_relative "../config/environment" +require "fileutils" +require "optparse" + +options = { release: "2026-08-27" } +OptionParser.new do |parser| + parser.on("--release VERSION") { options[:release] = _1 } + parser.on("--output PATH") { options[:output] = Pathname(_1).expand_path } + parser.on("--provinces LIST") { options[:provinces] = _1.split(",").map(&:strip) } +end.parse! +abort "missing option: --output" unless options[:output] +abort "refusing to overwrite #{options[:output]}" if options[:output].exist? + +release = Warehouse::InstitutionRelease.find_by!(version: options.fetch(:release)) +payload = Warehouse::FinancialStatementExtraction::CoverageAudit.new( + release:, provinces: options[:provinces] +).payload + +FileUtils.mkdir_p(options[:output].dirname) +options[:output].write(JSON.pretty_generate(payload) << "\n") +puts JSON.pretty_generate(payload.except(:records).merge(output: options[:output].to_s)) diff --git a/script/audit_prairie_parser_upgrade.rb b/script/audit_prairie_parser_upgrade.rb new file mode 100644 index 00000000..091a12df --- /dev/null +++ b/script/audit_prairie_parser_upgrade.rb @@ -0,0 +1,93 @@ +#!/usr/bin/env ruby + +require_relative "../config/environment" +require "optparse" + +options = { + release: "2026-08-27", provinces: [ "sk" ], + from_parser: "prairie-municipal-form-v1", limit: nil +} +OptionParser.new do |parser| + parser.on("--release VERSION") { options[:release] = _1 } + parser.on("--provinces LIST") { options[:provinces] = _1.split(",").map(&:strip) } + parser.on("--from-parser VERSION") { options[:from_parser] = _1 } + parser.on("--start-extraction-id ID", Integer) { options[:start_extraction_id] = _1 } + parser.on("--limit COUNT", Integer) { options[:limit] = _1 } + parser.on("--only-fallback-labels") { options[:only_fallback_labels] = true } +end.parse! + +release = Warehouse::InstitutionRelease.find_by!(version: options.fetch(:release)) +candidates = Warehouse::FinancialStatementExtraction::CandidateSet.new( + release:, provinces: options.fetch(:provinces) +) +candidate_by_key = candidates.each.index_by { [ _1.asset_sha256, _1.fiscal_year_end ] } +scope = release.financial_statement_extractions.where( + extractor_version: Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION, + status: "approved" +).where("llm_response_snapshot ->> 'parser' = ?", options.fetch(:from_parser)).order(:id) +province_clauses = options.fetch(:provinces).map { "institution_canonical_id LIKE ?" }.join(" OR ") +scope = scope.where(province_clauses, *options.fetch(:provinces).map { "ca/#{_1}/%" }) +scope = scope.where("id > ?", options[:start_extraction_id]) if options[:start_extraction_id] +if options[:only_fallback_labels] + risky_ids = Warehouse::FinancialStatementFact.where( + raw_label: [ "FINANCIAL ASSETS", "LIABILITIES", "NON-FINANCIAL ASSETS" ] + ).select(:financial_statement_extraction_id) + scope = scope.where(id: risky_ids) +end +scope = scope.limit(options[:limit]) if options[:limit] +counts = Hash.new(0) + +scope.find_each do |extraction| + candidate = candidate_by_key.fetch([ extraction.asset_sha256, extraction.fiscal_year_end ]) + result = Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline.new( + pdf_path: candidate.pdf_path, + institution_canonical_id: candidate.institution_canonical_id, + institution_name: candidate.institution_name, + document_canonical_id: candidate.document_canonical_id, + asset_sha256: candidate.asset_sha256, + fiscal_year_end: candidate.fiscal_year_end, + population: candidate.population + ).run + stored_facts = extraction.financial_statement_facts.to_h { [ _1.concept, _1.value ] } + reparsed_facts = result.facts.to_h { [ _1.fetch(:concept), _1.fetch(:value) ] } + line_item_signature = lambda do |item| + attributes = item.respond_to?(:attributes) ? item.attributes.symbolize_keys : item + attributes.slice(:flow, :category, :label, :value, :scale, :source_page, :column_year, :position) + .merge(value: attributes.fetch(:value).to_d) + end + sort_line_items = ->(items) { items.sort_by { [ _1.fetch(:flow), _1.fetch(:position) ] } } + stored_line_items = sort_line_items.call( + extraction.financial_statement_line_items.map { line_item_signature.call(_1) } + ) + reparsed_line_items = sort_line_items.call(result.line_items.map { line_item_signature.call(_1) }) + facts_match = stored_facts == reparsed_facts + line_items_match = stored_line_items == reparsed_line_items + status = facts_match && line_items_match ? "pass" : "mismatch" + counts[status] += 1 + puts({ + extraction_id: extraction.id, + document: extraction.document_canonical_id, + from_parser: options.fetch(:from_parser), + to_parser: Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline::PARSER_VERSION, + status:, + facts_match:, + line_items_match:, + stored_facts: stored_facts.transform_values(&:to_s), + reparsed_facts: reparsed_facts.transform_values(&:to_s), + stored_line_items: stored_line_items.map { _1.merge(value: _1.fetch(:value).to_s("F")) }, + reparsed_line_items: reparsed_line_items.map { _1.merge(value: _1.fetch(:value).to_s("F")) }, + verification_check_count: result.checks.length + }.to_json) +rescue => error + counts["failed"] += 1 + puts({ + extraction_id: extraction.id, + document: extraction.document_canonical_id, + from_parser: options.fetch(:from_parser), + to_parser: Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline::PARSER_VERSION, + status: "failed", + error: "#{error.class}: #{error.message}" + }.to_json) +end + +puts({ release: release.version, summary: counts }.to_json) diff --git a/script/enqueue_municipal_financial_extractions.rb b/script/enqueue_municipal_financial_extractions.rb new file mode 100644 index 00000000..7b6621cf --- /dev/null +++ b/script/enqueue_municipal_financial_extractions.rb @@ -0,0 +1,42 @@ +#!/usr/bin/env ruby + +require_relative "../config/environment" +require "optparse" + +options = { provinces: Warehouse::FinancialStatementExtraction::CandidateSet::PROVINCES, rerun: "missing" } +OptionParser.new do |parser| + parser.banner = "Usage: bin/rails runner script/enqueue_municipal_financial_extractions.rb --release VERSION [options]" + parser.on("--release VERSION") { options[:release] = _1 } + parser.on("--provinces LIST") { options[:provinces] = _1.split(",").map(&:strip) } + parser.on("--years LIST") { options[:years] = _1.split(",").map { Integer(it) } } + parser.on("--institution-ids LIST") { options[:institution_ids] = _1.split(",").map(&:strip) } + parser.on("--limit COUNT", Integer) { options[:limit] = _1 } + parser.on("--rerun POLICY", Warehouse::FinancialStatementExtraction::Processor::RERUN_POLICIES) { options[:rerun] = _1 } + parser.on("--asset-root PATH") { options[:asset_root] = Pathname(_1).expand_path.to_s } + parser.on("--dry-run") { options[:dry_run] = true } + parser.on("--verify-hashes") { options[:verify_hashes] = true } +end.parse! +abort "missing option: release" unless options[:release] + +release = Warehouse::InstitutionRelease.find_by!(version: options.fetch(:release)) +results = options.fetch(:provinces).to_h do |province| + candidate_options = { + release:, provinces: [ province ], years: options[:years], + institution_ids: options[:institution_ids] + } + candidate_options[:asset_root] = options[:asset_root] if options[:asset_root] + candidates = Warehouse::FinancialStatementExtraction::CandidateSet.new(**candidate_options) + if options[:dry_run] + audit = candidates.audit(verify_hashes: options[:verify_hashes]) + [ province, audit ] + else + job_options = { + province:, years: options[:years], institution_ids: options[:institution_ids], + limit: options[:limit], rerun: options.fetch(:rerun), asset_root: options[:asset_root] + }.compact + job = Warehouse::ExtractMunicipalFinancialStatementsJob.perform_later(release.version, **job_options) + [ province, { candidates: candidates.count, job_id: job.job_id } ] + end +end + +puts JSON.pretty_generate(release: release.version, dry_run: options[:dry_run] || false, provinces: results) diff --git a/script/import_census_profile_population.rb b/script/import_census_profile_population.rb new file mode 100644 index 00000000..299bde58 --- /dev/null +++ b/script/import_census_profile_population.rb @@ -0,0 +1,17 @@ +#!/usr/bin/env ruby + +require_relative "../config/environment" +require "optparse" + +options = { retrieved_at: Time.current } +OptionParser.new do |parser| + parser.banner = "Usage: bin/rails runner script/import_census_profile_population.rb --zip PATH --sha256 SHA --retrieved-at ISO8601" + parser.on("--zip PATH") { |value| options[:zip_path] = value } + parser.on("--sha256 SHA") { |value| options[:expected_sha256] = value } + parser.on("--retrieved-at ISO8601") { |value| options[:retrieved_at] = Time.iso8601(value) } +end.parse! +missing = %i[zip_path expected_sha256].reject { |key| options[key].present? } +abort "missing options: #{missing.join(', ')}" if missing.any? + +result = Warehouse::CensusProfileImporter.new(**options).import! +puts JSON.pretty_generate(result) diff --git a/script/import_municipal_financial_pilot.rb b/script/import_municipal_financial_pilot.rb new file mode 100644 index 00000000..f491fb17 --- /dev/null +++ b/script/import_municipal_financial_pilot.rb @@ -0,0 +1,72 @@ +#!/usr/bin/env ruby + +require_relative "../config/environment" +require "optparse" + +options = {} +OptionParser.new do |parser| + parser.banner = "Usage: bin/rails runner script/import_municipal_financial_pilot.rb --release VERSION --config PATH --audit PATH" + parser.on("--release VERSION") { |value| options[:release] = value } + parser.on("--config PATH") { |value| options[:config] = Pathname(value).expand_path } + parser.on("--audit PATH") { |value| options[:audit] = Pathname(value).expand_path } +end.parse! +missing = %i[release config audit].reject { |key| options[key].present? } +abort "missing options: #{missing.join(', ')}" if missing.any? + +release = Warehouse::InstitutionRelease.find_by!(version: options.fetch(:release)) +config = JSON.parse(options.fetch(:config).read) +audit = JSON.parse(options.fetch(:audit).read).fetch("results").index_by { |row| row.fetch("city") } +imported = [] +skipped = [] + +config.fetch("entries").each do |entry| + city = entry.fetch("city") + validation = audit.fetch(city) + unless validation.fetch("current_status") == "extracted" && validation.fetch("failed_checks").empty? + raise "#{city} did not pass the pinned validator audit" + end + + payload = JSON.parse(Pathname(entry.fetch("extraction_path")).read) + asset = release.institution_document_assets.find_by(content_sha256: payload.fetch("asset_sha256")) + unless asset + skipped << { city:, reason: "asset is absent from the final release" } + next + end + document = asset.institution_document + extraction = release.financial_statement_extractions.find_or_initialize_by( + asset_sha256: payload.fetch("asset_sha256"), extractor_version: payload.fetch("extractor_version") + ) + next imported << city if extraction.status == "approved" + + extraction.assign_attributes( + institution_canonical_id: document.institution.canonical_id, + document_canonical_id: document.canonical_id, + fiscal_year_end: Date.iso8601(payload.fetch("fiscal_year_end")), + statement_basis: payload.fetch("statement_basis"), language: payload.fetch("language"), + llm_model: payload.fetch("model"), status: "extracted", check_results: payload.fetch("checks"), + llm_response_snapshot: payload.fetch("model_response") + ) + extraction.transaction do + extraction.save! + extraction.financial_statement_facts.delete_all + payload.fetch("facts").each do |fact| + extraction.financial_statement_facts.create!( + concept: fact.fetch("concept"), value: BigDecimal(fact.fetch("value")), + raw_text: fact.fetch("raw_text"), raw_label: fact.fetch("raw_label"), + scale: fact.fetch("scale"), statement: fact.fetch("statement"), + source_page: fact.fetch("source_page"), column_year: fact.fetch("column_year"), + extraction_confidence: fact.fetch("extraction_confidence") + ) + end + extraction.approve!( + reviewer: "local-pilot-validator", + notes: "Local preview only; imported from pinned independent validator audit #{options.fetch(:audit)}" + ) + end + imported << city +end + +puts JSON.pretty_generate( + release: release.version, imported_cities: imported, imported_count: imported.length, + skipped: skipped, skipped_count: skipped.length +) diff --git a/script/process_municipal_financial_details.rb b/script/process_municipal_financial_details.rb new file mode 100644 index 00000000..4e64af91 --- /dev/null +++ b/script/process_municipal_financial_details.rb @@ -0,0 +1,97 @@ +#!/usr/bin/env ruby + +require_relative "../config/environment" +require "optparse" + +options = {} +OptionParser.new do |parser| + parser.banner = "Usage: bin/rails runner script/process_municipal_financial_details.rb --release VERSION --config PATH [--cities city,city] [--years 2024,2025]" + parser.on("--release VERSION") { |value| options[:release] = value } + parser.on("--config PATH") { |value| options[:config] = Pathname(value).expand_path } + parser.on("--cities LIST") { |value| options[:cities] = value.split(",").map(&:strip) } + parser.on("--years LIST") { |value| options[:years] = value.split(",").map { Integer(_1) } } +end.parse! +missing = %i[release config].reject { |key| options[key].present? } +abort "missing options: #{missing.join(', ')}" if missing.any? + +release = Warehouse::InstitutionRelease.find_by!(version: options.fetch(:release)) +entries = JSON.parse(options.fetch(:config).read).fetch("entries") +entries.select! { |entry| entry.fetch("city").in?(options.fetch(:cities)) } if options[:cities] +entries.select! do |entry| + year = Date.iso8601(entry["fiscal_year_end"] || JSON.parse(Pathname(entry.fetch("extraction_path")).read).fetch("fiscal_year_end")).year + year.in?(options.fetch(:years)) +end if options[:years] +results = [] + +entries.each do |entry| + city = entry.fetch("city") + payload = entry["extraction_path"] ? JSON.parse(Pathname(entry.fetch("extraction_path")).read) : entry + fiscal_year_end = Date.iso8601(payload.fetch("fiscal_year_end")) + asset_sha256 = payload.fetch("asset_sha256") + document_canonical_id = payload.fetch("document_canonical_id") + institution_canonical_id = payload["institution_canonical_id"] || + release.institution_documents.find_by!(canonical_id: document_canonical_id).institution.canonical_id + institution = release.institutions.find_by!(canonical_id: institution_canonical_id) + + headline = release.financial_statement_extractions.find_or_initialize_by( + asset_sha256:, + extractor_version: Warehouse::FinancialStatementExtraction::Pipeline::EXTRACTOR_VERSION, + fiscal_year_end: + ) + unless headline.status == "approved" + headline.assign_attributes( + institution_canonical_id:, + document_canonical_id:, + fiscal_year_end:, + statement_basis: "consolidated", + llm_model: payload["model"] || Warehouse::FinancialStatementExtraction::Pipeline::DEFAULT_MODEL, + status: "pending" + ) + headline.save! + headline_result = headline.extractor.extract( + pdf_path: entry.fetch("pdf_path"), institution_name: institution.name_en, + population: entry["population"] + ) + unless headline_result.status == "extracted" + results << { city:, fiscal_year: fiscal_year_end.year, status: headline.reload.status, + reason: "headline extraction did not pass all deterministic checks" } + next + end + end + + extraction = release.financial_statement_extractions.find_or_initialize_by( + asset_sha256: headline.asset_sha256, + extractor_version: Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION, + fiscal_year_end: + ) + if extraction.status == "approved" + results << { city:, fiscal_year: fiscal_year_end.year, status: "approved", + line_items: extraction.financial_statement_line_items.count } + next + end + extraction.assign_attributes( + institution_canonical_id: headline.institution_canonical_id, + document_canonical_id: headline.document_canonical_id, + fiscal_year_end: headline.fiscal_year_end, + statement_basis: headline.statement_basis, + language: headline.language, + llm_model: Warehouse::FinancialStatementExtraction::DetailedPipeline::DEFAULT_MODEL, + status: "pending" + ) + extraction.save! + result = extraction.extractor.extract_detailed( + pdf_path: entry.fetch("pdf_path"), + institution_name: institution.name_en, + population: entry["population"] + ) + results << { + city:, fiscal_year: fiscal_year_end.year, status: extraction.reload.status, + line_items: extraction.financial_statement_line_items.count, + failed_checks: Array(extraction.check_results).count { |check| check["status"] == "fail" } + } +rescue => error + results << { city:, fiscal_year: fiscal_year_end&.year, status: "failed", + error: "#{error.class}: #{error.message}" } +end + +puts JSON.pretty_generate(release: release.version, results:) diff --git a/script/process_municipal_financial_statements.rb b/script/process_municipal_financial_statements.rb new file mode 100644 index 00000000..11897f58 --- /dev/null +++ b/script/process_municipal_financial_statements.rb @@ -0,0 +1,102 @@ +#!/usr/bin/env ruby + +require_relative "../config/environment" +require "optparse" +require "set" + +options = { release: "2026-08-27", rerun: "missing", limit: nil, start: nil, stop_before: nil } +OptionParser.new do |parser| + parser.banner = "Usage: script/process_municipal_financial_statements.rb --province CODE [options]" + parser.on("--release VERSION") { options[:release] = _1 } + parser.on("--province CODE") { options[:province] = _1.downcase } + parser.on("--years LIST") { options[:years] = _1.split(",").map { Integer(it) } } + parser.on("--institutions LIST") { options[:institution_ids] = _1.split(",").map(&:strip) } + parser.on("--rerun POLICY") { options[:rerun] = _1 } + parser.on("--limit COUNT", Integer) { options[:limit] = _1 } + parser.on("--start DOCUMENT_DATABASE_ID", Integer) { options[:start] = _1 } + parser.on("--stop-before DOCUMENT_DATABASE_ID", Integer) { options[:stop_before] = _1 } + parser.on("--document-ids LIST") { options[:document_ids] = _1.split(",").map { Integer(it) } } + parser.on("--exclude-document-ids LIST") do |value| + options[:excluded_document_ids] = value.split(",").map { Integer(it) } + end + parser.on("--failed-only") { options[:failed_only] = true } + parser.on("--failed-extractor TARGET", %w[headline detailed]) do |value| + options[:failed_extractor] = value + end + parser.on("--failed-parser LIST") do |value| + options[:failed_parser_versions] ||= [] + options[:failed_parser_versions].concat(value.split(",").map(&:strip).reject(&:empty?)) + end +end.parse! +abort "missing option: province" unless options[:province] +abort "--failed-only requires --rerun failed" if options[:failed_only] && options[:rerun] != "failed" +abort "--failed-extractor requires --failed-only" if options[:failed_extractor] && !options[:failed_only] +if options[:failed_parser_versions]&.any? && !options[:failed_only] + abort "--failed-parser requires --failed-only" +end +if options[:failed_extractor] == "headline" && options[:failed_parser_versions]&.any? + abort "--failed-parser only supports the detailed failed extractor" +end + +release = Warehouse::InstitutionRelease.find_by!(version: options.fetch(:release)) +candidates = Warehouse::FinancialStatementExtraction::CandidateSet.new( + release:, provinces: [ options.fetch(:province) ], years: options[:years], + institution_ids: options[:institution_ids] +) +abort "asset root is unavailable: #{candidates.asset_root}" unless candidates.asset_root.directory? + +processor = Warehouse::FinancialStatementExtraction::Processor.new(release:, rerun: options.fetch(:rerun)) +candidate_window = Warehouse::FinancialStatementExtraction::CandidateWindow.new( + start: options[:start], stop_before: options[:stop_before], + document_ids: options[:document_ids], excluded_document_ids: options[:excluded_document_ids] +) +candidate_rows = options[:failed_only] ? candidates.each.to_a : candidates.each(start: options[:start]) +failed_filter = if options[:failed_only] + failed_extractor_version = if options[:failed_extractor] == "headline" + Warehouse::FinancialStatementExtraction::Pipeline::EXTRACTOR_VERSION + else + Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION + end + Warehouse::FinancialStatementExtraction::FailedCandidateFilter.new( + release:, province: options.fetch(:province), candidates: candidate_rows, + parser_versions: options[:failed_parser_versions], failed_extractor_version: + ) +end +if failed_filter + puts({ failed_only_scope: failed_filter.report }.to_json) + abort "failed-only scope has unmatched persisted failures" if failed_filter.unmatched_keys.any? +end +counts = Hash.new(0) +processed = 0 +consecutive_source_failures = 0 +processed_failed_keys = Set.new + +candidate_rows.each do |candidate| + next if candidate_window.before_start?(candidate) + break if candidate_window.at_or_after_stop?(candidate) + next unless candidate_window.selected?(candidate) + next if candidate_window.excluded?(candidate) + break if options[:limit] && processed >= options[:limit] + if failed_filter + next unless failed_filter.eligible?(candidate) + next unless processed_failed_keys.add?(failed_filter.key_for(candidate)) + end + + result = processor.call(candidate) + processed += 1 + counts[result.status] += 1 + consecutive_source_failures = if result.status.in?(%w[missing_asset concurrent_skip]) + consecutive_source_failures + 1 + else + 0 + end + puts({ + index: processed, document_id: candidate.document_id, + document: candidate.document_canonical_id, fiscal_year: candidate.fiscal_year_end.year, + status: result.status, stage: result.stage, extraction_id: result.extraction_id, + error: result.error + }.compact.to_json) + abort "source access circuit opened after three consecutive failures" if consecutive_source_failures >= 3 +end + +puts({ summary: counts, processed:, failed_only: options[:failed_only] || false }.to_json) diff --git a/script/process_quebec_financial_forms.rb b/script/process_quebec_financial_forms.rb new file mode 100755 index 00000000..0721c803 --- /dev/null +++ b/script/process_quebec_financial_forms.rb @@ -0,0 +1,36 @@ +#!/usr/bin/env ruby + +require_relative "../config/environment" +require "optparse" + +options = { release: "2026-08-27", limit: nil, start: nil, stop_before: nil } +OptionParser.new do |parser| + parser.banner = "Usage: script/process_quebec_financial_forms.rb [options]" + parser.on("--release VERSION") { options[:release] = _1 } + parser.on("--limit COUNT", Integer) { options[:limit] = _1 } + parser.on("--start DOCUMENT_DATABASE_ID", Integer) { options[:start] = _1 } + parser.on("--stop-before DOCUMENT_DATABASE_ID", Integer) { options[:stop_before] = _1 } +end.parse! + +release = Warehouse::InstitutionRelease.find_by!(version: options.fetch(:release)) +candidates = Warehouse::FinancialStatementExtraction::CandidateSet.new(release:, provinces: [ "qc" ]) +abort "asset root is unavailable: #{candidates.asset_root}" unless candidates.asset_root.directory? +processor = Warehouse::FinancialStatementExtraction::QuebecFormProcessor.new(release:) +counts = Hash.new(0) +processed = 0 +consecutive_missing_assets = 0 + +candidates.each(start: options[:start]) do |candidate| + break if options[:stop_before] && candidate.document_id >= options[:stop_before] + break if options[:limit] && processed >= options[:limit] + + result = processor.call(candidate) + processed += 1 + counts[result.status] += 1 + consecutive_missing_assets = result.status == "missing_asset" ? consecutive_missing_assets + 1 : 0 + puts({ index: processed, status: result.status, document: result.document_canonical_id, + extraction_id: result.detailed_extraction_id, error: result.error }.compact.to_json) + abort "asset root appears unavailable after three consecutive missing PDFs" if consecutive_missing_assets >= 3 +end + +puts({ summary: counts, processed: }.to_json) diff --git a/script/process_saskatchewan_financial_forms.rb b/script/process_saskatchewan_financial_forms.rb new file mode 100755 index 00000000..eeee9ce5 --- /dev/null +++ b/script/process_saskatchewan_financial_forms.rb @@ -0,0 +1,58 @@ +#!/usr/bin/env ruby + +require_relative "../config/environment" +require "optparse" +require "set" + +options = { release: "2026-08-27", province: "sk", limit: nil, start: nil, stop_before: nil } +OptionParser.new do |parser| + parser.banner = "Usage: script/process_saskatchewan_financial_forms.rb [options]" + parser.on("--release VERSION") { options[:release] = _1 } + parser.on("--province CODE") { options[:province] = _1.downcase } + parser.on("--limit COUNT", Integer) { options[:limit] = _1 } + parser.on("--start DOCUMENT_DATABASE_ID", Integer) { options[:start] = _1 } + parser.on("--stop-before DOCUMENT_DATABASE_ID", Integer) { options[:stop_before] = _1 } + parser.on("--failed-only") { options[:failed_only] = true } +end.parse! + +release = Warehouse::InstitutionRelease.find_by!(version: options.fetch(:release)) +candidates = Warehouse::FinancialStatementExtraction::CandidateSet.new(release:, provinces: [ options[:province] ]) +abort "asset root is unavailable: #{candidates.asset_root}" unless candidates.asset_root.directory? +processor = Warehouse::FinancialStatementExtraction::SaskatchewanFormProcessor.new(release:) +candidate_window = Warehouse::FinancialStatementExtraction::CandidateWindow.new( + start: options[:start], stop_before: options[:stop_before] +) +candidate_rows = candidates.each.to_a +failed_filter = if options[:failed_only] + Warehouse::FinancialStatementExtraction::FailedCandidateFilter.new( + release:, province: options.fetch(:province), candidates: candidate_rows + ) +end +if failed_filter + puts({ failed_only_scope: failed_filter.report }.to_json) + abort "failed-only scope has unmatched persisted failures" if failed_filter.unmatched_keys.any? +end +counts = Hash.new(0) +processed = 0 +consecutive_missing_assets = 0 +processed_failed_keys = Set.new + +candidate_rows.each do |candidate| + next if candidate_window.before_start?(candidate) + break if candidate_window.at_or_after_stop?(candidate) + break if options[:limit] && processed >= options[:limit] + if failed_filter + next unless failed_filter.eligible?(candidate) + next unless processed_failed_keys.add?(failed_filter.key_for(candidate)) + end + + result = processor.call(candidate) + processed += 1 + counts[result.status] += 1 + consecutive_missing_assets = result.status == "missing_asset" ? consecutive_missing_assets + 1 : 0 + puts({ index: processed, status: result.status, document: result.document_canonical_id, + extraction_id: result.detailed_extraction_id, error: result.error }.compact.to_json) + abort "asset root appears unavailable after three consecutive missing PDFs" if consecutive_missing_assets >= 3 +end + +puts({ summary: counts, processed:, failed_only: options[:failed_only] || false }.to_json) diff --git a/script/revalidate_municipal_financial_headlines.rb b/script/revalidate_municipal_financial_headlines.rb new file mode 100644 index 00000000..fb5b8392 --- /dev/null +++ b/script/revalidate_municipal_financial_headlines.rb @@ -0,0 +1,49 @@ +#!/usr/bin/env ruby + +require_relative "../config/environment" +require "optparse" + +options = { release: "2026-08-27", provinces: [], limit: nil } +OptionParser.new do |parser| + parser.banner = "Usage: script/revalidate_municipal_financial_headlines.rb [options]" + parser.on("--release VERSION") { options[:release] = _1 } + parser.on("--provinces x,y,z", Array) { options[:provinces] = _1.map(&:downcase) } + parser.on("--limit COUNT", Integer) { options[:limit] = _1 } +end.parse! + +release = Warehouse::InstitutionRelease.find_by!(version: options.fetch(:release)) +candidates = Warehouse::FinancialStatementExtraction::CandidateSet.new( + release:, provinces: options[:provinces].presence +) +abort "asset root is unavailable: #{candidates.asset_root}" unless candidates.asset_root.directory? +processed = 0 +counts = Hash.new(0) + +candidates.each do |candidate| + break if options[:limit] && processed >= options[:limit] + + extraction = release.financial_statement_extractions.find_by( + asset_sha256: candidate.asset_sha256, + extractor_version: Warehouse::FinancialStatementExtraction::Pipeline::EXTRACTOR_VERSION, + fiscal_year_end: candidate.fiscal_year_end, + status: "needs_review" + ) + next unless extraction + next if extraction.llm_response_snapshot.blank? + + result = extraction.extractor.revalidate_headline( + pdf_path: candidate.pdf_path, + institution_name: candidate.institution_name, + population: candidate.population + ) + processed += 1 + counts[result.status] += 1 + puts({ id: extraction.id, document: extraction.document_canonical_id, status: result.status }.to_json) +rescue => error + processed += 1 + counts["failed"] += 1 + puts({ id: extraction&.id, document: candidate.document_canonical_id, + status: "failed", error: "#{error.class}: #{error.message}" }.compact.to_json) +end + +puts({ release: release.version, processed:, summary: counts }.to_json) diff --git a/script/review_extracted_municipal_financial_statements.rb b/script/review_extracted_municipal_financial_statements.rb new file mode 100644 index 00000000..52256df2 --- /dev/null +++ b/script/review_extracted_municipal_financial_statements.rb @@ -0,0 +1,132 @@ +#!/usr/bin/env ruby + +require_relative "../config/environment" +require "optparse" + +options = { batch_size: 100, idle_rounds: 12 } +OptionParser.new do |parser| + parser.banner = "Usage: bin/rails runner script/review_extracted_municipal_financial_statements.rb --release VERSION [options]" + parser.on("--release VERSION") { options[:release] = _1 } + parser.on("--provinces LIST") { options[:provinces] = _1.split(",").map(&:strip) } + parser.on("--limit COUNT", Integer) { options[:limit] = _1 } + parser.on("--batch-size COUNT", Integer) { options[:batch_size] = _1 } + parser.on("--shard INDEX/COUNT") do |value| + options[:shard_index], options[:shard_count] = value.split("/", 2).map { Integer(_1) } + end + parser.on("--watch") { options[:watch] = true } + parser.on("--idle-rounds COUNT", Integer) { options[:idle_rounds] = _1 } + parser.on("--asset-root PATH") { options[:asset_root] = Pathname(_1).expand_path.to_s } + parser.on("--dry-run") { options[:dry_run] = true } + parser.on("--audit-approved-by REVIEWER") { options[:audit_approved_by] = _1 } + parser.on("--promote-audit-approved-by REVIEWER") { options[:promote_audit_approved_by] = _1 } + parser.on("--retry-needs-review") { options[:retry_needs_review] = true } + parser.on("--parser-version VERSION") { options[:parser_version] = _1 } +end.parse! +abort "missing option: release" unless options[:release] +abort "--audit-approved-by cannot be combined with --watch" if options[:audit_approved_by] && options[:watch] +if options[:promote_audit_approved_by] && + options.values_at(:watch, :dry_run, :audit_approved_by, :retry_needs_review).any? + abort "--promote-audit-approved-by cannot be combined with watch, dry-run, audit, or retry modes" +end +unsupported_provinces = Array(options[:provinces]) - Warehouse::FinancialStatementExtraction::CandidateSet::PROVINCES +abort "unsupported provinces: #{unsupported_provinces.join(', ')}" if unsupported_provinces.any? + +release = Warehouse::InstitutionRelease.find_by!(version: options.fetch(:release)) +if options[:shard_count] + abort "shard index must be between zero and count minus one" unless options[:shard_index]&.between?(0, options[:shard_count] - 1) +end + +reviewed = 0 +promoted = 0 +idle_rounds = 0 +last_attempted_id = 0 +loop do + audit_reviewer = options[:audit_approved_by] || options[:promote_audit_approved_by] + statuses = if audit_reviewer + [ "approved" ] + elsif options[:retry_needs_review] + %w[extracted needs_review] + else + [ "extracted" ] + end + scope = release.financial_statement_extractions.where( + extractor_version: Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION, + status: statuses + ).order(:id) + if audit_reviewer || options[:retry_needs_review] + scope = scope.where("id > ?", last_attempted_id) + end + scope = scope.where(reviewed_by: audit_reviewer) if audit_reviewer + if options[:parser_version] + scope = scope.where("llm_response_snapshot ->> 'parser' = ?", options[:parser_version]) + end + if options[:provinces] + clauses = options[:provinces].map { "institution_canonical_id LIKE ?" }.join(" OR ") + scope = scope.where(clauses, *options[:provinces].map { "ca/#{_1}/%" }) + end + if options[:shard_count] + scope = scope.where("MOD(id, ?) = ?", options[:shard_count], options[:shard_index]) + end + remaining = options[:limit] && options[:limit] - reviewed + break if remaining && remaining <= 0 + + batch_limit = [ options[:batch_size], remaining ].compact.min + batch = ActiveRecord::Base.uncached { scope.limit(batch_limit).to_a } + if batch.empty? + break unless options[:watch] + + idle_rounds += 1 + break if idle_rounds >= options[:idle_rounds] + + sleep 5 + next + end + + idle_rounds = 0 + batch.each do |extraction| + row = begin + if options[:dry_run] + { id: extraction.id, institution: extraction.institution_canonical_id, + fiscal_year: extraction.fiscal_year_end.year, status: "reviewable" } + else + reviewer_options = { extraction: } + reviewer_options[:asset_root] = options[:asset_root] if options[:asset_root] + reviewer = Warehouse::FinancialStatementExtraction::Reviewer.new(**reviewer_options) + if options[:promote_audit_approved_by] + previous = { + reviewed_by: extraction.reviewed_by, reviewed_at: extraction.reviewed_at&.iso8601, + check_results: extraction.check_results + } + result = reviewer.reaudit! + extraction.reload + promoted += 1 if extraction.reviewed_by.in?( + Warehouse::FinancialStatementExtraction::Reviewer::DETERMINISTIC_REVIEWERS + ) + { id: extraction.id, institution: extraction.institution_canonical_id, + fiscal_year: extraction.fiscal_year_end.year, status: result.status, + previous:, reviewed_by: extraction.reviewed_by, + reviewed_at: extraction.reviewed_at&.iso8601, checks: result.checks } + else + result = options[:audit_approved_by] ? reviewer.audit : reviewer.review! + { id: extraction.id, institution: extraction.institution_canonical_id, + fiscal_year: extraction.fiscal_year_end.year, status: result.status } + end + end + rescue => error + unless extraction.reviewed_at? || audit_reviewer + extraction.update!(status: "needs_review", error_message: "review error: #{error.class}: #{error.message}") + end + { id: extraction.id, institution: extraction.institution_canonical_id, + fiscal_year: extraction.fiscal_year_end.year, status: "error", + error: "#{error.class}: #{error.message}" } + end + reviewed += 1 + last_attempted_id = extraction.id + puts row.to_json + end + + break if options[:dry_run] +end + +puts({ release: release.version, dry_run: options[:dry_run] || false, + promote_audit: options[:promote_audit_approved_by].present?, reviewed:, promoted: }.to_json) diff --git a/script/run_atlantic_financial_handoff.zsh b/script/run_atlantic_financial_handoff.zsh new file mode 100644 index 00000000..5f74f6ed --- /dev/null +++ b/script/run_atlantic_financial_handoff.zsh @@ -0,0 +1,105 @@ +#!/bin/zsh + +set -e +set -o pipefail + +cd "${0:A:h}/.." + +output_root=/Volumes/floppy/york_factory/public_institutions/financial-extractions +ocr_cache_root=$output_root/ocr-cache-v1 +handoff_log=$output_root/atlantic-nb-pe-after-mb-v2-2026-08-30.log + +test ! -e "$handoff_log" +exec > >(tee "$handoff_log") 2>&1 +export PGHOST=127.0.0.1 PGPORT=55434 PGUSER=brendansamek MUNICIPAL_FINANCIAL_OCR_CACHE_ROOT=$ocr_cache_root + +reviewer_alive() { + tmux list-panes -t "$1" -F '#{pane_dead}' 2>/dev/null | rg -qx '0' +} + +drain_review() { + province="$1" + reviewer_session="$2" + polls=0 + while true; do + state=$(bin/rails runner "release=Warehouse::InstitutionRelease.find_by!(version:'2026-08-27'); scope=release.financial_statement_extractions.where('institution_canonical_id LIKE ?', 'ca/${province}/%'); active=scope.where(status:%w[pending extracting]).count; extracted=scope.where(extractor_version:Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION,status:'extracted').count; puts \"#{active}:#{extracted}\"" | tail -n 1) + active="${state%%:*}" + extracted="${state##*:}" + test "$active" = "0" || { echo "$province active rows after extractor: $active"; exit 1; } + if test "$extracted" = "0"; then + echo "$province reviewer drain complete" + return + fi + reviewer_alive "$reviewer_session" || { echo "$reviewer_session died with $extracted extracted rows"; exit 1; } + polls=$((polls + 1)) + test "$polls" -lt 720 || { echo "$province reviewer drain timed out with $extracted extracted rows"; exit 1; } + echo "$province waiting for reviewer: $extracted extracted rows" + sleep 30 + done +} + +assert_terminal_checks() { + bin/rails runner 'terminal=%w[extracted needs_review approved rejected failed]; bad=Warehouse::FinancialStatementExtraction.where(status:terminal).where("jsonb_typeof(check_results) <> ? OR jsonb_array_length(check_results)=0","array").count; abort "terminal rows without checks: #{bad}" unless bad.zero?; puts({terminal_without_checks:0}.to_json)' +} + +current_load() { + sysctl -n vm.loadavg | awk '{ print $2 }' +} + +load_monitor_pid= +stop_load_monitor() { + if test -n "$load_monitor_pid"; then + kill "$load_monitor_pid" 2>/dev/null || true + wait "$load_monitor_pid" 2>/dev/null || true + load_monitor_pid= + fi +} + +start_load_monitor() { + label="$1" + ( + while sleep 60; do + load=$(current_load) + if awk -v load="$load" 'BEGIN { exit !(load > 15) }'; then + echo "{\"load_alert\":\"$label\",\"one_minute_load\":$load,\"action\":\"alert_only\",\"recovery\":\"SIGTERM the producer, run the logged stale-extracting sweep, then relaunch this coordinator\"}" + fi + done + ) & + load_monitor_pid=$! +} + +trap stop_load_monitor EXIT INT TERM + +startup_load=$(current_load) +if awk -v load="$startup_load" 'BEGIN { exit !(load > 15) }'; then + echo "Atlantic early-start load gate failed: one-minute load $startup_load exceeds 15" + exit 3 +fi + +assert_terminal_checks +bin/rails runner 'release=Warehouse::InstitutionRelease.find_by!(version:"2026-08-27"); totals=Warehouse::FinancialStatementExtraction::CoverageAudit.new(release:,provinces:["nb"]).payload.fetch(:totals); expected={"unattempted"=>240,"shared_asset"=>9}; abort "NB prelane changed: #{totals.fetch(:status_counts).inspect}" unless totals.fetch(:status_counts)==expected; scope=release.financial_statement_extractions.where("institution_canonical_id LIKE ?","ca/nb/%"); active=scope.where(status:%w[pending extracting]).count; abort "NB active rows: #{active}" unless active.zero?; puts({preflight:"pass",province:"nb",statuses:expected}.to_json)' +tmux has-session -t municipal-nb-review-watch-v1 2>/dev/null && { echo "NB reviewer already exists"; exit 1; } +tmux new-session -d -s municipal-nb-review-watch-v1 -c "$PWD" "set -o pipefail; PGHOST=127.0.0.1 PGPORT=55434 PGUSER=brendansamek MUNICIPAL_FINANCIAL_OCR_CACHE_ROOT=$ocr_cache_root MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=1 bin/rails runner script/review_extracted_municipal_financial_statements.rb --release 2026-08-27 --provinces nb --retry-needs-review --batch-size 25 --watch --idle-rounds 1440 2>&1 | tee $output_root/nb-review-watch-v1-2026-08-30.jsonl" +start_load_monitor nb +MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=1 bin/rails runner script/process_municipal_financial_statements.rb --release 2026-08-27 --province nb --rerun missing 2>&1 | tee $output_root/nb-full-missing-v1-2026-08-30.jsonl +stop_load_monitor +drain_review nb municipal-nb-review-watch-v1 +assert_terminal_checks +bin/rails runner script/audit_municipal_financial_extraction_coverage.rb --release 2026-08-27 --provinces nb --output $output_root/nb-final-coverage-v1-2026-08-30.json + +bin/rails runner 'release=Warehouse::InstitutionRelease.find_by!(version:"2026-08-27"); totals=Warehouse::FinancialStatementExtraction::CoverageAudit.new(release:,provinces:["pe"]).payload.fetch(:totals); expected={"headline_pending"=>3,"headline_needs_review"=>10,"needs_review"=>1,"unattempted"=>165}; abort "PE prelane changed: #{totals.fetch(:status_counts).inspect}" unless totals.fetch(:status_counts)==expected; detailed=release.financial_statement_extractions.where("institution_canonical_id LIKE ?","ca/pe/%").where(extractor_version:Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION,status:%w[pending extracting]).count; abort "PE detailed active rows: #{detailed}" unless detailed.zero?; puts({preflight:"pass",province:"pe",statuses:expected,detailed_active:0}.to_json)' +tmux has-session -t municipal-pe-review-watch-v1 2>/dev/null && { echo "PE reviewer already exists"; exit 1; } +tmux new-session -d -s municipal-pe-review-watch-v1 -c "$PWD" "set -o pipefail; PGHOST=127.0.0.1 PGPORT=55434 PGUSER=brendansamek MUNICIPAL_FINANCIAL_OCR_CACHE_ROOT=$ocr_cache_root MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=1 bin/rails runner script/review_extracted_municipal_financial_statements.rb --release 2026-08-27 --provinces pe --retry-needs-review --batch-size 25 --watch --idle-rounds 1440 2>&1 | tee $output_root/pe-review-watch-v1-2026-08-30.jsonl" +start_load_monitor pe +MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=1 bin/rails runner script/process_municipal_financial_statements.rb --release 2026-08-27 --province pe --rerun missing 2>&1 | tee $output_root/pe-full-missing-v1-2026-08-30.jsonl +stop_load_monitor +drain_review pe municipal-pe-review-watch-v1 +assert_terminal_checks +bin/rails runner script/audit_municipal_financial_extraction_coverage.rb --release 2026-08-27 --provinces pe --output $output_root/pe-final-coverage-v1-2026-08-30.json + +while kill -0 8058 2>/dev/null; do + sleep 30 +done +drain_review mb municipal-mb-review-watch-v1 +assert_terminal_checks +echo "Atlantic NB/PE handoff complete" diff --git a/script/run_national_financial_finalization.zsh b/script/run_national_financial_finalization.zsh new file mode 100644 index 00000000..58e48933 --- /dev/null +++ b/script/run_national_financial_finalization.zsh @@ -0,0 +1,246 @@ +#!/bin/zsh + +set -e +set -o pipefail + +cd "${0:A:h}/.." + +output_root=/Volumes/floppy/york_factory/public_institutions/financial-extractions +ocr_cache_root=$output_root/ocr-cache-v1 +run_stamp=2026-08-30 +coordinator_log=$output_root/national-finalization-v1-$run_stamp.log +legacy_dry_run=$output_root/national-legacy-reviewer-dry-run-v1-$run_stamp.jsonl +legacy_promotion=$output_root/national-legacy-reviewer-promotion-v1-$run_stamp.jsonl +coverage_output=$output_root/national-final-per-record-verification-v1-$run_stamp.json +york_test_log=$output_root/national-final-york-tests-v1-$run_stamp.log +york_quality_log=$output_root/national-final-york-quality-v1-$run_stamp.log +frontend_test_log=$output_root/national-final-canadaspends-tests-v1-$run_stamp.log +frontend_build_log=$output_root/national-final-canadaspends-build-v1-$run_stamp.log +local_api_result=$output_root/national-final-local-toronto-api-v1-$run_stamp.json +local_frontend_result=$output_root/national-final-local-toronto-page-v1-$run_stamp.html +local_redirect_headers=$output_root/national-final-local-toronto-redirect-v1-$run_stamp.headers +tunnel_frontend_result=$output_root/national-final-tunnel-toronto-page-v1-$run_stamp.html +tunnel_redirect_headers=$output_root/national-final-tunnel-toronto-redirect-v1-$run_stamp.headers +transient_retry_provinces=(nl mb nb pe on) +transient_retry_logs=() +for retry_province in $transient_retry_provinces; do + transient_retry_logs+=( + $output_root/$retry_province-national-headline-retry-v1-$run_stamp.jsonl + $output_root/$retry_province-national-detailed-retry-v1-$run_stamp.jsonl + $output_root/$retry_province-national-retry-review-v1-$run_stamp.jsonl + ) +done + +outputs=( + $coordinator_log $legacy_dry_run $legacy_promotion $coverage_output + $york_test_log $york_quality_log $frontend_test_log $frontend_build_log + $local_api_result $local_frontend_result $local_redirect_headers + $tunnel_frontend_result $tunnel_redirect_headers + $transient_retry_logs +) +for output_path in $outputs; do + test ! -e "$output_path" || { echo "refusing existing output: $output_path"; exit 1; } +done +exec > >(tee "$coordinator_log") 2>&1 +export PGHOST=127.0.0.1 PGPORT=55434 PGUSER=brendansamek MUNICIPAL_FINANCIAL_OCR_CACHE_ROOT=$ocr_cache_root + +dependencies=( + municipal-territories-after-nl-v1 + municipal-atlantic-nb-pe-after-mb-v2 + municipal-ns-bc-after-atlantic-v2 + municipal-on-final-after-bounded-lanes-v2 + municipal-qc-generic-fallback-after-on-v1 + municipal-prairie-v3-after-stable-v3 +) + +polls=0 +while true; do + all_dead=1 + for session in $dependencies; do + dead=$(tmux display-message -p -t "$session":0 '#{pane_dead}' 2>/dev/null) || { + echo "$session disappeared" + exit 1 + } + if test "$dead" = "1"; then + exit_status=$(tmux display-message -p -t "$session":0 '#{pane_dead_status}') + test "$exit_status" = "0" || { + echo "$session failed with status $exit_status; finalization intentionally halted" + exit 1 + } + else + all_dead=0 + fi + done + test "$all_dead" = "0" || break + polls=$((polls + 1)) + test "$polls" -lt 40320 || { echo "national dependency wait timed out"; exit 1; } + sleep 30 +done +echo "All provincial and territorial coordinators completed cleanly" + +# Generic-first provinces get one slot-guarded retry for both headline and detailed failures. +# Headline failures must be selected separately because they have no persisted detailed row. +for retry_province in $transient_retry_provinces; do + headline_retry_log=$output_root/$retry_province-national-headline-retry-v1-$run_stamp.jsonl + detailed_retry_log=$output_root/$retry_province-national-detailed-retry-v1-$run_stamp.jsonl + retry_review_log=$output_root/$retry_province-national-retry-review-v1-$run_stamp.jsonl + + MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=2 MUNICIPAL_FINANCIAL_DETAIL_FLOW_CONCURRENCY=2 \ + bin/rails runner script/process_municipal_financial_statements.rb \ + --release 2026-08-27 --province "$retry_province" --rerun failed --failed-only \ + --failed-extractor headline 2>&1 | tee "$headline_retry_log" + MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=2 MUNICIPAL_FINANCIAL_DETAIL_FLOW_CONCURRENCY=2 \ + bin/rails runner script/process_municipal_financial_statements.rb \ + --release 2026-08-27 --province "$retry_province" --rerun failed --failed-only \ + --failed-extractor detailed 2>&1 | tee "$detailed_retry_log" + MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=1 \ + bin/rails runner script/review_extracted_municipal_financial_statements.rb \ + --release 2026-08-27 --provinces "$retry_province" --batch-size 25 \ + 2>&1 | tee "$retry_review_log" + + PROVINCE="$retry_province" bin/rails runner ' + release = Warehouse::InstitutionRelease.find_by!(version: "2026-08-27") + province = ENV.fetch("PROVINCE") + scope = release.financial_statement_extractions.where( + "institution_canonical_id LIKE ?", "ca/#{province}/%" + ) + detailed = scope.where( + extractor_version: Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION + ) + headline = scope.where( + extractor_version: Warehouse::FinancialStatementExtraction::Pipeline::EXTRACTOR_VERSION + ) + detailed_active = detailed.where(status: %w[pending extracting extracted]).count + headline_active = headline.where(status: %w[pending extracting]).count + terminal = %w[extracted needs_review approved rejected failed] + without_checks = Warehouse::FinancialStatementExtraction.where(status: terminal) + .where("jsonb_typeof(check_results) <> ? OR jsonb_array_length(check_results)=0", "array").count + abort "#{province.upcase} detailed rows undrained after retry: #{detailed_active}" unless detailed_active.zero? + abort "#{province.upcase} headline rows active after retry: #{headline_active}" unless headline_active.zero? + abort "terminal rows without checks after #{province.upcase} retry: #{without_checks}" unless without_checks.zero? + puts({ province:, detailed_active: 0, headline_active: 0, terminal_without_checks: 0 }.to_json) + ' +done +echo "Generic-first provincial headline and detailed retry sweep completed" + +# Review watchers are no longer allowed to mutate data once the extraction lanes have drained. +reviewer_sessions=( + municipal-mb-review-watch-v1 municipal-nb-review-watch-v1 municipal-pe-review-watch-v1 + municipal-on-review-shard0-v1 municipal-on-review-shard1-v1 + municipal-sk-stable-review municipal-ab-stable-review + municipal-ab-prairie-v3-review municipal-sk-prairie-v3-review + municipal-bc-stable-review municipal-nl-retry-review-after-v2 + municipal-qc-generic-canary-review-v1 municipal-qc-generic-remainder-review-v1 +) +for session in $reviewer_sessions; do + tmux kill-session -t "$session" 2>/dev/null || true +done + +if pgrep -fl 'process_(municipal_financial_statements|quebec_financial_forms|saskatchewan_financial_forms)\.rb' > /tmp/national-financial-extraction-processes.txt; then + cat /tmp/national-financial-extraction-processes.txt + echo "extraction process still running; frozen finalization halted" + exit 1 +fi + +preflight=$(bin/rails runner 'release=Warehouse::InstitutionRelease.find_by!(version:"2026-08-27"); version=Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION; release_scope=release.financial_statement_extractions; detailed=release_scope.where(extractor_version:version); active=detailed.where(status:%w[pending extracting extracted]).count; bad_terminal=Warehouse::FinancialStatementExtraction.where(status:%w[extracted needs_review approved rejected failed]).where("jsonb_typeof(check_results) <> ? OR jsonb_array_length(check_results)=0","array").count; deterministic=Warehouse::FinancialStatementExtraction::Reviewer::DETERMINISTIC_REVIEWERS; legacy=detailed.where(status:"approved").where("reviewed_by IS NULL OR reviewed_by NOT IN (?)",deterministic).group(:reviewed_by).count; abort "national active detailed rows: #{active}" unless active.zero?; abort "terminal rows without checks: #{bad_terminal}" unless bad_terminal.zero?; abort "unexpected legacy reviewers: #{legacy.inspect}" unless (legacy.keys.compact-["local-detailed-validator"]).empty? && !legacy.key?(nil); puts({active_detailed:0,terminal_without_checks:0,legacy_reviewers:legacy}.to_json)' | tail -n 1) +echo "$preflight" +legacy_count=$(echo "$preflight" | ruby -rjson -e 'puts JSON.parse(STDIN.read).fetch("legacy_reviewers").fetch("local-detailed-validator",0)') + +bin/rails runner script/review_extracted_municipal_financial_statements.rb \ + --release 2026-08-27 --audit-approved-by local-detailed-validator --dry-run \ + 2>&1 | tee "$legacy_dry_run" +dry_count=$(tail -n 1 "$legacy_dry_run" | ruby -rjson -e 'puts JSON.parse(STDIN.read).fetch("reviewed")') +test "$dry_count" = "$legacy_count" || { + echo "legacy dry-run count drift: expected=$legacy_count actual=$dry_count" + exit 1 +} + +if test "$legacy_count" -gt 0; then + bin/rails runner script/review_extracted_municipal_financial_statements.rb \ + --release 2026-08-27 --promote-audit-approved-by local-detailed-validator \ + 2>&1 | tee "$legacy_promotion" +else + echo '{"release":"2026-08-27","promote_audit":true,"reviewed":0,"promoted":0}' | tee "$legacy_promotion" +fi +promotion_summary=$(tail -n 1 "$legacy_promotion") +promoted=$(echo "$promotion_summary" | ruby -rjson -e 'puts JSON.parse(STDIN.read).fetch("promoted")') +reviewed=$(echo "$promotion_summary" | ruby -rjson -e 'puts JSON.parse(STDIN.read).fetch("reviewed")') +test "$reviewed" = "$legacy_count" && test "$promoted" = "$legacy_count" || { + echo "legacy promotion incomplete: expected=$legacy_count reviewed=$reviewed promoted=$promoted" + exit 1 +} + +bin/rails runner script/audit_municipal_financial_extraction_coverage.rb \ + --release 2026-08-27 --output "$coverage_output" +COVERAGE_OUTPUT="$coverage_output" bin/rails runner 'payload=JSON.parse(File.read(ENV.fetch("COVERAGE_OUTPUT"))); totals=payload.fetch("totals"); statuses=totals.fetch("status_counts"); forbidden=%w[unattempted pending extracting extracted headline_pending headline_extracting headline_extracted]; present=forbidden.to_h { [_1,statuses.fetch(_1,0)] }.reject { |_key,value| value.zero? }; gates={approved_without_checks:totals.fetch("approved_without_checks"),failed_headline_gate_without_checks:totals.fetch("failed_headline_gate_without_checks"),shared_asset_with_terminal_extraction_without_checks:totals.fetch("shared_asset_with_terminal_extraction_without_checks"),approved_without_deterministic_reviewer:totals.fetch("approved_without_deterministic_reviewer")}; abort "unresolved coverage statuses: #{present.inspect}" if present.any?; abort "coverage verification gates failed: #{gates.inspect}" unless gates.values.all?(&:zero?); records=payload.fetch("records"); missing=records.select { |row| row.fetch("status")!="shared_asset" && row.dig("verification","total").to_i.zero? && row.fetch("status").in?(%w[approved rejected failed needs_review failed_headline_gate]) }; abort "terminal coverage records without saved results: #{missing.take(10).map { _1.fetch("document_canonical_id") }.inspect}" if missing.any?; puts({preferred_assets:totals.fetch("preferred_asset_count"),institution_years:totals.fetch("institution_year_count"),published_institution_years:totals.fetch("published_institution_year_count"),status_counts:statuses,verification_gates:gates}.to_json)' + +{ + bundle exec ruby test/scripts/sanitize_municipal_report_batch_test.rb + bin/rails test \ + test/controllers/api/v1/warehouse/municipal_financial_statements_controller_test.rb \ + test/jobs/warehouse/extract_municipal_financial_statements_job_test.rb \ + test/models/warehouse/census_profile_importer_test.rb \ + test/models/warehouse/financial_statement_extraction_test.rb \ + test/models/warehouse/financial_statement_extraction/*.rb \ + test/scripts/audit_municipal_financial_extraction_coverage_test.rb +} 2>&1 | tee "$york_test_log" + +{ + bundle exec rubocop \ + app/controllers/api/v1/warehouse/municipal_financial_statements_controller.rb \ + app/jobs/warehouse/extract_municipal_financial_statements_job.rb \ + app/models/warehouse/financial_statement_extraction.rb \ + app/models/warehouse/financial_statement_extraction \ + app/models/warehouse/financial_statement_line_item.rb \ + app/models/warehouse/census_profile.rb app/models/warehouse/census_profile_importer.rb \ + script/audit_municipal_financial_extraction_coverage.rb \ + script/process_municipal_financial_statements.rb \ + script/review_extracted_municipal_financial_statements.rb \ + script/sanitize_municipal_report_batch.rb \ + test/scripts/sanitize_municipal_report_batch_test.rb + bin/rails zeitwerk:check +} 2>&1 | tee "$york_quality_log" + +( + cd ../CanadaSpends + pnpm test 2>&1 | tee "$frontend_test_log" + YORK_FACTORY_API_URL=http://127.0.0.1:3001/api/v1 pnpm build \ + 2>&1 | tee "$frontend_build_log" +) + +curl --fail --silent --show-error \ + http://127.0.0.1:3001/api/v1/warehouse/municipal_financial_statements/on/toronto/2025 \ + -o "$local_api_result" +API_RESULT="$local_api_result" ruby -rjson -e 'payload=JSON.parse(File.read(ENV.fetch("API_RESULT"))); statement=payload.fetch("statements").find { _1.fetch("fiscal_year")==2025 } or abort "Toronto 2025 absent"; verification=statement.fetch("verification"); abort "saved checks absent" unless verification.dig("summary","total").to_i.positive? && verification.fetch("checks").any?; sankey=statement.fetch("sankey"); revenue_children=sankey.dig("revenue_data","children"); spending_children=sankey.dig("spending_data","children"); abort "Sankey absent" unless sankey.fetch("revenue").positive? && sankey.fetch("spending").positive? && revenue_children&.any? && spending_children&.any?; abort "context absent" unless payload.dig("context","population").to_i.positive?; puts({toronto_latest:payload.fetch("available_years").max,checks:verification.dig("summary","total"),sankey_revenue_groups:revenue_children.length,sankey_spending_groups:spending_children.length,population:payload.dig("context","population")}.to_json)' + +curl --fail --silent --show-error --head \ + --dump-header "$local_redirect_headers" --output /dev/null \ + http://127.0.0.1:3200/en/municipal/on/toronto +rg -q '^HTTP/[^ ]+ 307' "$local_redirect_headers" +rg -qi '^location: /en/municipal/on/toronto/2025\r?$' "$local_redirect_headers" + +curl --fail --silent --show-error http://127.0.0.1:3200/en/municipal/on/toronto/2025 \ + -o "$local_frontend_result" +rg -q 'View all [0-9]+ verification checks' "$local_frontend_result" +rg -q 'Inflows|Outflows' "$local_frontend_result" + +tunnel_session=$(tmux list-sessions -F '#{session_name}' | rg '^municipal-cloudflare-tunnel' | tail -n 1) +test -n "$tunnel_session" || { echo "Cloudflare tunnel session not found"; exit 1; } +tunnel_log=/tmp/$tunnel_session.log +tunnel_url=$({ + tmux capture-pane -pt "$tunnel_session" -S - + test ! -f "$tunnel_log" || sed -n '1,240p' "$tunnel_log" +} | rg -o 'https://[a-z0-9-]+\.trycloudflare\.com' | tail -n 1) +test -n "$tunnel_url" || { echo "Cloudflare tunnel URL not found"; exit 1; } +curl --fail --silent --show-error --head \ + --dump-header "$tunnel_redirect_headers" --output /dev/null \ + "$tunnel_url/en/municipal/on/toronto" +rg -q '^HTTP/[^ ]+ 307' "$tunnel_redirect_headers" +rg -qi '^location: /en/municipal/on/toronto/2025\r?$' "$tunnel_redirect_headers" + +curl --fail --silent --show-error "$tunnel_url/en/municipal/on/toronto/2025" \ + -o "$tunnel_frontend_result" +rg -q 'View all [0-9]+ verification checks' "$tunnel_frontend_result" +rg -q 'Inflows|Outflows' "$tunnel_frontend_result" + +echo "National financial extraction finalization and local/tunnel QA complete: $tunnel_url" diff --git a/script/run_ns_bc_financial_handoff.zsh b/script/run_ns_bc_financial_handoff.zsh new file mode 100644 index 00000000..e26f84ec --- /dev/null +++ b/script/run_ns_bc_financial_handoff.zsh @@ -0,0 +1,111 @@ +#!/bin/zsh + +set -e +set -o pipefail + +cd "${0:A:h}/.." + +output_root=/Volumes/floppy/york_factory/public_institutions/financial-extractions +ocr_cache_root=$output_root/ocr-cache-v1 +coordinator_log=$output_root/ns-bc-after-atlantic-v2-2026-08-30.log +outputs=( + $output_root/ns-generic-failed-v1-2026-08-30.jsonl + $output_root/ns-generic-review-watch-v1-2026-08-30.jsonl + $output_root/ns-generic-finite-review-v1-2026-08-30.jsonl + $output_root/ns-generic-final-coverage-v1-2026-08-30.json + $output_root/bc-generic-failed-v1-2026-08-30.jsonl + $output_root/bc-generic-review-watch-v1-2026-08-30.jsonl + $output_root/bc-generic-finite-review-v1-2026-08-30.jsonl + $output_root/bc-generic-final-coverage-v1-2026-08-30.json +) + +test ! -e "$coordinator_log" +for output_path in $outputs; do + test ! -e "$output_path" || { echo "refusing existing output: $output_path"; exit 1; } +done + +exec > >(tee "$coordinator_log") 2>&1 +export PGHOST=127.0.0.1 PGPORT=55434 PGUSER=brendansamek MUNICIPAL_FINANCIAL_OCR_CACHE_ROOT=$ocr_cache_root MUNICIPAL_FINANCIAL_DETAIL_FLOW_CONCURRENCY=2 + +polls=0 +while true; do + dead=$(tmux display-message -p -t municipal-atlantic-nb-pe-after-mb-v2:0 '#{pane_dead}' 2>/dev/null) || { + echo "Atlantic coordinator disappeared" + exit 1 + } + if test "$dead" = "1"; then + exit_status=$(tmux display-message -p -t municipal-atlantic-nb-pe-after-mb-v2:0 '#{pane_dead_status}') + test "$exit_status" = "0" || { + echo "Atlantic coordinator failed with status $exit_status" + exit 1 + } + break + fi + polls=$((polls + 1)) + test "$polls" -lt 20160 || { echo "Atlantic coordinator wait timed out"; exit 1; } + sleep 30 +done + +pane_alive() { + tmux list-panes -t "$1" -F '#{pane_dead}' 2>/dev/null | rg -qx '0' +} + +drain_province() { + province="$1" + reviewer_session="$2" + polls=0 + while true; do + state=$(bin/rails runner "release=Warehouse::InstitutionRelease.find_by!(version:'2026-08-27'); scope=release.financial_statement_extractions.where('institution_canonical_id LIKE ?', 'ca/${province}/%').where(extractor_version:Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION); active=scope.where(status:%w[pending extracting]).count; extracted=scope.where(status:'extracted').count; puts \"#{active}:#{extracted}\"" | tail -n 1) + active="${state%%:*}" + extracted="${state##*:}" + test "$active" = "0" || { echo "$province active rows after extractor: $active"; exit 1; } + if test "$extracted" = "0"; then + echo "$province reviewer drain complete" + return + fi + pane_alive "$reviewer_session" || { echo "$reviewer_session died with $extracted extracted rows"; exit 1; } + polls=$((polls + 1)) + test "$polls" -lt 1440 || { echo "$province reviewer drain timed out with $extracted rows"; exit 1; } + echo "$province waiting for reviewer: $extracted extracted rows" + sleep 30 + done +} + +assert_terminal_checks() { + bin/rails runner 'terminal=%w[extracted needs_review approved rejected failed]; bad=Warehouse::FinancialStatementExtraction.where(status:terminal).where("jsonb_typeof(check_results) <> ? OR jsonb_array_length(check_results)=0","array").count; abort "terminal rows without checks: #{bad}" unless bad.zero?; puts({terminal_without_checks:0}.to_json)' +} + +assert_terminal_checks +ns_baseline=$(bin/rails runner 'release=Warehouse::InstitutionRelease.find_by!(version:"2026-08-27"); candidates=Warehouse::FinancialStatementExtraction::CandidateSet.new(release:,provinces:["ns"]).each.to_a; filter=Warehouse::FinancialStatementExtraction::FailedCandidateFilter.new(release:,province:"ns",candidates:); report=filter.report; abort "NS filter changed: #{report.slice(:public_slot_count,:unmatched_failure_count,:reconciled).inspect}" unless report.fetch(:public_slot_count)==355 && report.fetch(:unmatched_failure_count).zero? && report.fetch(:reconciled); totals=Warehouse::FinancialStatementExtraction::CoverageAudit.new(release:,provinces:["ns"]).payload.fetch(:totals); active=release.financial_statement_extractions.where("institution_canonical_id LIKE ?","ca/ns/%").where(extractor_version:Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION,status:%w[pending extracting extracted]).count; abort "NS active detailed rows: #{active}" unless active.zero?; puts totals.fetch(:published_institution_year_count)' | tail -n 1) +echo "NS preflight selected=355 baseline_published=$ns_baseline; new path=deterministic prairie-v3 with generic DetailedPipeline fallback on Unsupported" + +tmux has-session -t municipal-ns-generic-review-v1 2>/dev/null && { echo "NS reviewer exists"; exit 1; } +tmux new-session -d -s municipal-ns-generic-review-v1 -c "$PWD" "set -o pipefail; PGHOST=127.0.0.1 PGPORT=55434 PGUSER=brendansamek MUNICIPAL_FINANCIAL_OCR_CACHE_ROOT=$ocr_cache_root MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=1 bin/rails runner script/review_extracted_municipal_financial_statements.rb --release 2026-08-27 --provinces ns --batch-size 25 --watch --idle-rounds 25920 2>&1 | tee $output_root/ns-generic-review-watch-v1-2026-08-30.jsonl" +MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=2 bin/rails runner script/process_municipal_financial_statements.rb --release 2026-08-27 --province ns --rerun failed --failed-only 2>&1 | tee $output_root/ns-generic-failed-v1-2026-08-30.jsonl +drain_province ns municipal-ns-generic-review-v1 +tmux kill-session -t municipal-ns-generic-review-v1 2>/dev/null || true +MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=1 bin/rails runner script/review_extracted_municipal_financial_statements.rb --release 2026-08-27 --provinces ns --retry-needs-review --batch-size 25 2>&1 | tee $output_root/ns-generic-finite-review-v1-2026-08-30.jsonl +assert_terminal_checks +bin/rails runner script/audit_municipal_financial_extraction_coverage.rb --release 2026-08-27 --provinces ns --output $output_root/ns-generic-final-coverage-v1-2026-08-30.json + +ns_gate=$(NS_BASELINE="$ns_baseline" bin/rails runner 'release=Warehouse::InstitutionRelease.find_by!(version:"2026-08-27"); totals=Warehouse::FinancialStatementExtraction::CoverageAudit.new(release:,provinces:["ns"]).payload.fetch(:totals); active=release.financial_statement_extractions.where("institution_canonical_id LIKE ?","ca/ns/%").where(extractor_version:Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION,status:%w[pending extracting extracted]).count; abort "NS undrained detailed rows: #{active}" unless active.zero?; published=totals.fetch(:published_institution_year_count); delta=published-Integer(ENV.fetch("NS_BASELINE")); needs=totals.fetch(:status_counts).fetch("needs_review",0); puts "#{published}:#{delta}:#{needs}"' | tail -n 1) +IFS=: read ns_published ns_delta ns_needs <<< "$ns_gate" +echo "NS gate published=$ns_published delta=$ns_delta needs_review=$ns_needs required_delta=71" +if test "$ns_delta" -lt 71; then + echo "NS gate failed; BC is intentionally not started; inspect NS and BC failure clusters" + exit 3 +fi + +bc_baseline=$(bin/rails runner 'release=Warehouse::InstitutionRelease.find_by!(version:"2026-08-27"); candidates=Warehouse::FinancialStatementExtraction::CandidateSet.new(release:,provinces:["bc"]).each.to_a; filter=Warehouse::FinancialStatementExtraction::FailedCandidateFilter.new(release:,province:"bc",candidates:); report=filter.report; abort "BC filter changed: #{report.slice(:public_slot_count,:unmatched_failure_count,:reconciled).inspect}" unless report.fetch(:public_slot_count)==1100 && report.fetch(:unmatched_failure_count).zero? && report.fetch(:reconciled); totals=Warehouse::FinancialStatementExtraction::CoverageAudit.new(release:,provinces:["bc"]).payload.fetch(:totals); active=release.financial_statement_extractions.where("institution_canonical_id LIKE ?","ca/bc/%").where(extractor_version:Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION,status:%w[pending extracting extracted]).count; abort "BC active detailed rows: #{active}" unless active.zero?; puts totals.fetch(:published_institution_year_count)' | tail -n 1) +echo "BC gate passed via NS; baseline_published=$bc_baseline selected=1100" + +tmux has-session -t municipal-bc-generic-review-v1 2>/dev/null && { echo "BC reviewer exists"; exit 1; } +tmux new-session -d -s municipal-bc-generic-review-v1 -c "$PWD" "set -o pipefail; PGHOST=127.0.0.1 PGPORT=55434 PGUSER=brendansamek MUNICIPAL_FINANCIAL_OCR_CACHE_ROOT=$ocr_cache_root MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=1 bin/rails runner script/review_extracted_municipal_financial_statements.rb --release 2026-08-27 --provinces bc --batch-size 25 --watch --idle-rounds 25920 2>&1 | tee $output_root/bc-generic-review-watch-v1-2026-08-30.jsonl" +MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=2 bin/rails runner script/process_municipal_financial_statements.rb --release 2026-08-27 --province bc --rerun failed --failed-only 2>&1 | tee $output_root/bc-generic-failed-v1-2026-08-30.jsonl +drain_province bc municipal-bc-generic-review-v1 +tmux kill-session -t municipal-bc-generic-review-v1 2>/dev/null || true +MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=1 bin/rails runner script/review_extracted_municipal_financial_statements.rb --release 2026-08-27 --provinces bc --retry-needs-review --batch-size 25 2>&1 | tee $output_root/bc-generic-finite-review-v1-2026-08-30.jsonl +assert_terminal_checks +bin/rails runner script/audit_municipal_financial_extraction_coverage.rb --release 2026-08-27 --provinces bc --output $output_root/bc-generic-final-coverage-v1-2026-08-30.json +bin/rails runner 'release=Warehouse::InstitutionRelease.find_by!(version:"2026-08-27"); scope=release.financial_statement_extractions.where("institution_canonical_id LIKE ?","ca/bc/%").where(extractor_version:Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION); active=scope.where(status:%w[pending extracting extracted]).count; abort "BC undrained detailed rows: #{active}" unless active.zero?; puts({bc_detailed_pending_extracting_extracted:0}.to_json)' +echo "NS and BC generic fallback cycles complete" diff --git a/script/run_on_final_financial_handoff.zsh b/script/run_on_final_financial_handoff.zsh new file mode 100644 index 00000000..daaff0ba --- /dev/null +++ b/script/run_on_final_financial_handoff.zsh @@ -0,0 +1,46 @@ +#!/bin/zsh + +set -e +set -o pipefail + +cd "${0:A:h}/.." + +output_root=/Volumes/floppy/york_factory/public_institutions/financial-extractions +ocr_cache_root=$output_root/ocr-cache-v1 +coordinator_log=$output_root/on-final-after-bounded-lanes-v2-2026-08-30.log +finite_review_log=$output_root/on-finite-review-final-v1-2026-08-30.jsonl +coverage_output=$output_root/on-final-coverage-v1-2026-08-30.json + +test ! -e "$coordinator_log" +test ! -e "$finite_review_log" +test ! -e "$coverage_output" +exec > >(tee "$coordinator_log") 2>&1 +export PGHOST=127.0.0.1 PGPORT=55434 PGUSER=brendansamek MUNICIPAL_FINANCIAL_OCR_CACHE_ROOT=$ocr_cache_root + +polls=0 +while true; do + all_dead=1 + for session in municipal-on-head-bounded-v2 municipal-on-tail-after-nl; do + dead=$(tmux display-message -p -t "$session":0 '#{pane_dead}' 2>/dev/null) || { + echo "$session disappeared" + exit 1 + } + if test "$dead" = "1"; then + exit_status=$(tmux display-message -p -t "$session":0 '#{pane_dead_status}') + test "$exit_status" = "0" || { echo "$session failed with status $exit_status"; exit 1; } + else + all_dead=0 + fi + done + test "$all_dead" = "0" || break + polls=$((polls + 1)) + test "$polls" -lt 8640 || { echo "Ontario bounded extractors timed out"; exit 1; } + sleep 30 +done + +bin/rails runner 'release=Warehouse::InstitutionRelease.find_by!(version:"2026-08-27"); scope=release.financial_statement_extractions.where("institution_canonical_id LIKE ?","ca/on/%").where(extractor_version:Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION); active=scope.where(status:%w[pending extracting]).count; abort "ON detailed pending/extracting after lanes: #{active}" unless active.zero?; puts({on_detailed_pending_extracting:0}.to_json)' +tmux kill-session -t municipal-on-review-watch-v2 2>/dev/null || true +MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=1 bin/rails runner script/review_extracted_municipal_financial_statements.rb --release 2026-08-27 --provinces on --retry-needs-review --batch-size 25 2>&1 | tee "$finite_review_log" +bin/rails runner 'release=Warehouse::InstitutionRelease.find_by!(version:"2026-08-27"); scope=release.financial_statement_extractions.where("institution_canonical_id LIKE ?","ca/on/%").where(extractor_version:Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION); active=scope.where(status:%w[pending extracting extracted]).count; abort "ON undrained detailed rows: #{active}" unless active.zero?; terminal=%w[extracted needs_review approved rejected failed]; bad=Warehouse::FinancialStatementExtraction.where(status:terminal).where("jsonb_typeof(check_results) <> ? OR jsonb_array_length(check_results)=0","array").count; abort "terminal rows without checks: #{bad}" unless bad.zero?; puts({on_detailed_pending_extracting_extracted:0,terminal_without_checks:0}.to_json)' +bin/rails runner script/audit_municipal_financial_extraction_coverage.rb --release 2026-08-27 --provinces on --output "$coverage_output" +echo "Ontario bounded extraction and finite review complete" diff --git a/script/run_prairie_v3_financial_handoff.zsh b/script/run_prairie_v3_financial_handoff.zsh new file mode 100644 index 00000000..98efd9de --- /dev/null +++ b/script/run_prairie_v3_financial_handoff.zsh @@ -0,0 +1,217 @@ +#!/bin/zsh + +set -e +set -o pipefail + +cd "${0:A:h}/.." + +output_root=/Volumes/floppy/york_factory/public_institutions/financial-extractions +ocr_cache_root=$output_root/ocr-cache-v1 +coordinator_log=$output_root/prairie-v3-after-stable-v3-2026-08-30.log +ab_required_generic_success_percent=20 +outputs=( + $output_root/ab-prairie-v3-2026-08-30.jsonl + $output_root/sk-prairie-v3-2026-08-30.jsonl + $output_root/ab-prairie-v3-review-2026-08-30.jsonl + $output_root/sk-prairie-v3-review-2026-08-30.jsonl + $output_root/ab-prairie-generic-v1-2026-08-30.jsonl + $output_root/ab-prairie-generic-review-v1-2026-08-30.jsonl + $output_root/ab-prairie-generic-finite-review-v1-2026-08-30.jsonl + $output_root/sk-prairie-generic-v1-2026-08-30.jsonl + $output_root/sk-prairie-generic-review-v1-2026-08-30.jsonl + $output_root/sk-prairie-generic-finite-review-v1-2026-08-30.jsonl + $output_root/ab-prairie-v3-final-coverage-2026-08-30.json + $output_root/sk-prairie-v3-final-coverage-2026-08-30.json +) + +test ! -e "$coordinator_log" +for output_path in $outputs; do + test ! -e "$output_path" || { echo "refusing existing output: $output_path"; exit 1; } +done +exec > >(tee "$coordinator_log") 2>&1 +export PGHOST=127.0.0.1 PGPORT=55434 PGUSER=brendansamek MUNICIPAL_FINANCIAL_OCR_CACHE_ROOT=$ocr_cache_root MUNICIPAL_FINANCIAL_DETAIL_FLOW_CONCURRENCY=2 + +while kill -0 65389 2>/dev/null || kill -0 65392 2>/dev/null; do + sleep 30 +done + +pane_alive() { + tmux list-panes -t "$1" -F '#{pane_dead}' 2>/dev/null | rg -qx '0' +} + +drain_parser() { + province="$1" + parser="$2" + reviewer_session="$3" + polls=0 + while true; do + state=$(bin/rails runner "release=Warehouse::InstitutionRelease.find_by!(version:'2026-08-27'); scope=release.financial_statement_extractions.where('institution_canonical_id LIKE ?', 'ca/${province}/%').where(extractor_version:Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION); active=scope.where(status:%w[pending extracting]).count; extracted=scope.where(status:'extracted').where(\"llm_response_snapshot ->> 'parser' = ?\", '${parser}').count; puts \"#{active}:#{extracted}\"" | tail -n 1) + active="${state%%:*}" + extracted="${state##*:}" + test "$active" = "0" || { echo "$province active rows after extractor: $active"; exit 1; } + if test "$extracted" = "0"; then + echo "$province $parser reviewer drain complete" + return + fi + pane_alive "$reviewer_session" || { echo "$reviewer_session died with $extracted extracted rows"; exit 1; } + polls=$((polls + 1)) + test "$polls" -lt 720 || { echo "$province $parser reviewer drain timed out with $extracted rows"; exit 1; } + echo "$province waiting for $parser reviewer: $extracted extracted rows" + sleep 30 + done +} + +assert_terminal_checks() { + bin/rails runner 'terminal=%w[extracted needs_review approved rejected failed]; bad=Warehouse::FinancialStatementExtraction.where(status:terminal).where("jsonb_typeof(check_results) <> ? OR jsonb_array_length(check_results)=0","array").count; abort "terminal rows without checks: #{bad}" unless bad.zero?; puts({terminal_without_checks:0}.to_json)' +} + +cleanup_generic_reviewers() { + tmux kill-session -t municipal-ab-prairie-generic-review-v1 2>/dev/null || true + tmux kill-session -t municipal-sk-prairie-generic-review-v1 2>/dev/null || true +} +trap cleanup_generic_reviewers EXIT INT TERM + +generic_preflight() { + province="$1" + PROVINCE="$province" REQUIRED_PERCENT="$ab_required_generic_success_percent" bin/rails runner ' + province = ENV.fetch("PROVINCE") + release = Warehouse::InstitutionRelease.find_by!(version: "2026-08-27") + version = Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION + scope = release.financial_statement_extractions.where( + "institution_canonical_id LIKE ?", "ca/#{province}/%" + ).where(extractor_version: version) + active = scope.where(status: %w[pending extracting extracted]).count + abort "#{province.upcase} active rows before generic retry: #{active}" unless active.zero? + allowed_parsers = %w[prairie-municipal-form-v2 prairie-municipal-form-v3] + foreign = scope.where(status: "failed").filter_map do |row| + parser = row.llm_response_snapshot&.fetch("parser", nil) + [row.id, parser] unless allowed_parsers.include?(parser) + end + abort "#{province.upcase} non-prairie failed rows: #{foreign.take(10).inspect}" if foreign.any? + candidates = Warehouse::FinancialStatementExtraction::CandidateSet.new( + release:, provinces: [province] + ).each.to_a + report = Warehouse::FinancialStatementExtraction::FailedCandidateFilter.new( + release:, province:, candidates: + ).report + abort "#{province.upcase} failed filter is not reconciled: #{report.slice(:unmatched_failure_count, :reconciled).inspect}" unless + report.fetch(:unmatched_failure_count).zero? && report.fetch(:reconciled) + totals = Warehouse::FinancialStatementExtraction::CoverageAudit.new( + release:, provinces: [province] + ).payload.fetch(:totals) + baseline = totals.fetch(:published_institution_year_count) + selected = report.fetch(:public_slot_count) + percent = Integer(ENV.fetch("REQUIRED_PERCENT")) + threshold = (selected * percent + 99) / 100 + puts [baseline, selected, threshold].join(":") + ' | tail -n 1 +} + +drain_generic() { + province="$1" + reviewer_session="$2" + polls=0 + while true; do + state=$(bin/rails runner "release=Warehouse::InstitutionRelease.find_by!(version:'2026-08-27'); scope=release.financial_statement_extractions.where('institution_canonical_id LIKE ?', 'ca/${province}/%').where(extractor_version:Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION); active=scope.where(status:%w[pending extracting]).count; extracted=scope.where(status:'extracted').count; puts \"#{active}:#{extracted}\"" | tail -n 1) + active="${state%%:*}" + extracted="${state##*:}" + test "$active" = "0" || { echo "$province active rows after generic extractor: $active"; exit 1; } + if test "$extracted" = "0"; then + echo "$province generic reviewer drain complete" + return + fi + pane_alive "$reviewer_session" || { echo "$reviewer_session died with $extracted extracted rows"; exit 1; } + polls=$((polls + 1)) + test "$polls" -lt 1440 || { echo "$province generic reviewer drain timed out with $extracted rows"; exit 1; } + echo "$province waiting for generic reviewer: $extracted extracted rows" + sleep 30 + done +} + +run_generic() { + province="$1" + reviewer_session="municipal-${province}-prairie-generic-review-v1" + extraction_log="$output_root/${province}-prairie-generic-v1-2026-08-30.jsonl" + reviewer_log="$output_root/${province}-prairie-generic-review-v1-2026-08-30.jsonl" + finite_log="$output_root/${province}-prairie-generic-finite-review-v1-2026-08-30.jsonl" + + tmux has-session -t "$reviewer_session" 2>/dev/null && { echo "$reviewer_session already exists"; exit 1; } + tmux new-session -d -s "$reviewer_session" -c "$PWD" "set -o pipefail; PGHOST=127.0.0.1 PGPORT=55434 PGUSER=brendansamek MUNICIPAL_FINANCIAL_OCR_CACHE_ROOT=$ocr_cache_root MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=1 bin/rails runner script/review_extracted_municipal_financial_statements.rb --release 2026-08-27 --provinces $province --batch-size 25 --watch --idle-rounds 25920 2>&1 | tee $reviewer_log" + MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=2 bin/rails runner script/process_municipal_financial_statements.rb \ + --release 2026-08-27 --province "$province" --rerun failed --failed-only \ + 2>&1 | tee "$extraction_log" + drain_generic "$province" "$reviewer_session" + tmux kill-session -t "$reviewer_session" 2>/dev/null || true + MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=1 bin/rails runner \ + script/review_extracted_municipal_financial_statements.rb \ + --release 2026-08-27 --provinces "$province" --retry-needs-review --batch-size 25 \ + 2>&1 | tee "$finite_log" + assert_terminal_checks +} + +drain_parser ab prairie-municipal-form-v2 municipal-ab-stable-review +drain_parser sk prairie-municipal-form-v2 municipal-sk-stable-review +assert_terminal_checks +tmux kill-session -t municipal-ab-stable-review 2>/dev/null || true +tmux kill-session -t municipal-sk-stable-review 2>/dev/null || true + +for session in municipal-ab-prairie-v3-review municipal-sk-prairie-v3-review municipal-ab-prairie-v3 municipal-sk-prairie-v3; do + tmux has-session -t "$session" 2>/dev/null && { echo "$session already exists"; exit 1; } +done + +tmux new-session -d -s municipal-ab-prairie-v3-review -c "$PWD" "set -o pipefail; PGHOST=127.0.0.1 PGPORT=55434 PGUSER=brendansamek MUNICIPAL_FINANCIAL_OCR_CACHE_ROOT=$ocr_cache_root MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=1 ruby script/review_extracted_municipal_financial_statements.rb --release 2026-08-27 --provinces ab --parser-version prairie-municipal-form-v3 --batch-size 25 --watch --idle-rounds 25920 2>&1 | tee $output_root/ab-prairie-v3-review-2026-08-30.jsonl" +tmux new-session -d -s municipal-sk-prairie-v3-review -c "$PWD" "set -o pipefail; PGHOST=127.0.0.1 PGPORT=55434 PGUSER=brendansamek MUNICIPAL_FINANCIAL_OCR_CACHE_ROOT=$ocr_cache_root MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=1 ruby script/review_extracted_municipal_financial_statements.rb --release 2026-08-27 --provinces sk --parser-version prairie-municipal-form-v3 --batch-size 25 --watch --idle-rounds 25920 2>&1 | tee $output_root/sk-prairie-v3-review-2026-08-30.jsonl" +tmux new-session -d -s municipal-ab-prairie-v3 -c "$PWD" "set -o pipefail; PGHOST=127.0.0.1 PGPORT=55434 PGUSER=brendansamek MUNICIPAL_FINANCIAL_OCR_CACHE_ROOT=$ocr_cache_root ruby script/process_saskatchewan_financial_forms.rb --release 2026-08-27 --province ab --failed-only 2>&1 | tee $output_root/ab-prairie-v3-2026-08-30.jsonl" +tmux set-option -t municipal-ab-prairie-v3 remain-on-exit on +tmux new-session -d -s municipal-sk-prairie-v3 -c "$PWD" "set -o pipefail; PGHOST=127.0.0.1 PGPORT=55434 PGUSER=brendansamek MUNICIPAL_FINANCIAL_OCR_CACHE_ROOT=$ocr_cache_root ruby script/process_saskatchewan_financial_forms.rb --release 2026-08-27 --province sk --failed-only 2>&1 | tee $output_root/sk-prairie-v3-2026-08-30.jsonl" +tmux set-option -t municipal-sk-prairie-v3 remain-on-exit on + +polls=0 +while true; do + all_dead=1 + for session in municipal-ab-prairie-v3 municipal-sk-prairie-v3; do + dead=$(tmux display-message -p -t "$session":0 '#{pane_dead}' 2>/dev/null) || { + echo "$session disappeared" + exit 1 + } + if test "$dead" = "1"; then + exit_status=$(tmux display-message -p -t "$session":0 '#{pane_dead_status}') + test "$exit_status" = "0" || { echo "$session failed with status $exit_status"; exit 1; } + else + all_dead=0 + fi + done + test "$all_dead" = "0" || break + polls=$((polls + 1)) + test "$polls" -lt 4320 || { echo "prairie v3 extractors timed out"; exit 1; } + sleep 30 +done + +drain_parser ab prairie-municipal-form-v3 municipal-ab-prairie-v3-review +drain_parser sk prairie-municipal-form-v3 municipal-sk-prairie-v3-review +assert_terminal_checks +tmux kill-session -t municipal-ab-prairie-v3-review 2>/dev/null || true +tmux kill-session -t municipal-sk-prairie-v3-review 2>/dev/null || true + +ab_gate=$(generic_preflight ab) +IFS=: read ab_baseline ab_selected ab_threshold <<< "$ab_gate" +echo "AB generic preflight baseline_published=$ab_baseline selected=$ab_selected required_delta=$ab_threshold percent=$ab_required_generic_success_percent" +run_generic ab +ab_published=$(bin/rails runner 'release=Warehouse::InstitutionRelease.find_by!(version:"2026-08-27"); totals=Warehouse::FinancialStatementExtraction::CoverageAudit.new(release:,provinces:["ab"]).payload.fetch(:totals); puts totals.fetch(:published_institution_year_count)' | tail -n 1) +ab_delta=$((ab_published - ab_baseline)) +echo "AB generic gate published=$ab_published delta=$ab_delta required_delta=$ab_threshold selected=$ab_selected" +if test "$ab_delta" -lt "$ab_threshold"; then + echo "AB generic gate failed; SK generic fallback intentionally not started" + exit 3 +fi + +sk_gate=$(generic_preflight sk) +IFS=: read sk_baseline sk_selected sk_threshold <<< "$sk_gate" +echo "SK generic preflight baseline_published=$sk_baseline selected=$sk_selected informational_threshold=$sk_threshold" +run_generic sk + +cleanup_generic_reviewers +trap - EXIT INT TERM +bin/rails runner script/audit_municipal_financial_extraction_coverage.rb --release 2026-08-27 --provinces ab --output $output_root/ab-prairie-v3-final-coverage-2026-08-30.json +bin/rails runner script/audit_municipal_financial_extraction_coverage.rb --release 2026-08-27 --provinces sk --output $output_root/sk-prairie-v3-final-coverage-2026-08-30.json +echo "Prairie deterministic and generic fallback handoff complete" diff --git a/script/run_qc_generic_fallback_handoff.zsh b/script/run_qc_generic_fallback_handoff.zsh new file mode 100644 index 00000000..30c04b53 --- /dev/null +++ b/script/run_qc_generic_fallback_handoff.zsh @@ -0,0 +1,110 @@ +#!/bin/zsh + +set -e +set -o pipefail + +cd "${0:A:h}/.." + +output_root=/Volumes/floppy/york_factory/public_institutions/financial-extractions +ocr_cache_root=$output_root/ocr-cache-v1 +manifest=$output_root/qc-generic-fallback-canary-manifest-v1-2026-08-30.json +coordinator_log=$output_root/qc-generic-fallback-after-on-v1-2026-08-30.log +canary_results=$output_root/qc-generic-fallback-canary-results-v1-2026-08-30.json +outputs=( + $output_root/qc-finite-review-before-fallback-v1-2026-08-30.jsonl + $output_root/qc-generic-fallback-canary-v1-2026-08-30.jsonl + $output_root/qc-generic-fallback-canary-review-watch-v1-2026-08-30.jsonl + $output_root/qc-generic-fallback-canary-finite-review-v1-2026-08-30.jsonl + $canary_results + $output_root/qc-generic-fallback-remainder-v1-2026-08-30.jsonl + $output_root/qc-generic-fallback-remainder-review-watch-v1-2026-08-30.jsonl + $output_root/qc-generic-fallback-remainder-finite-review-v1-2026-08-30.jsonl + $output_root/qc-generic-fallback-final-coverage-v1-2026-08-30.json +) + +test -f "$manifest" +test ! -e "$coordinator_log" +for output_path in $outputs; do + test ! -e "$output_path" || { echo "refusing existing output: $output_path"; exit 1; } +done +exec > >(tee "$coordinator_log") 2>&1 +export PGHOST=127.0.0.1 PGPORT=55434 PGUSER=brendansamek MUNICIPAL_FINANCIAL_OCR_CACHE_ROOT=$ocr_cache_root MUNICIPAL_FINANCIAL_DETAIL_FLOW_CONCURRENCY=2 + +polls=0 +while true; do + dead=$(tmux display-message -p -t municipal-on-final-after-bounded-lanes-v2:0 '#{pane_dead}' 2>/dev/null) || { + echo "Ontario final coordinator disappeared" + exit 1 + } + if test "$dead" = "1"; then + exit_status=$(tmux display-message -p -t municipal-on-final-after-bounded-lanes-v2:0 '#{pane_dead_status}') + test "$exit_status" = "0" || { echo "Ontario final coordinator failed with status $exit_status"; exit 1; } + break + fi + polls=$((polls + 1)) + test "$polls" -lt 20160 || { echo "Ontario final coordinator wait timed out"; exit 1; } + sleep 30 +done + +pane_alive() { + tmux list-panes -t "$1" -F '#{pane_dead}' 2>/dev/null | rg -qx '0' +} + +drain_qc() { + reviewer_session="$1" + polls=0 + while true; do + state=$(bin/rails runner 'release=Warehouse::InstitutionRelease.find_by!(version:"2026-08-27"); scope=release.financial_statement_extractions.where("institution_canonical_id LIKE ?","ca/qc/%").where(extractor_version:Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION); active=scope.where(status:%w[pending extracting]).count; extracted=scope.where(status:"extracted").count; puts "#{active}:#{extracted}"' | tail -n 1) + active="${state%%:*}" + extracted="${state##*:}" + test "$active" = "0" || { echo "QC active rows after extractor: $active"; exit 1; } + if test "$extracted" = "0"; then + echo "QC reviewer drain complete" + return + fi + pane_alive "$reviewer_session" || { echo "$reviewer_session died with $extracted extracted rows"; exit 1; } + polls=$((polls + 1)) + test "$polls" -lt 1440 || { echo "QC reviewer drain timed out with $extracted rows"; exit 1; } + echo "QC waiting for reviewer: $extracted extracted rows" + sleep 30 + done +} + +assert_terminal_checks() { + bin/rails runner 'terminal=%w[extracted needs_review approved rejected failed]; bad=Warehouse::FinancialStatementExtraction.where(status:terminal).where("jsonb_typeof(check_results) <> ? OR jsonb_array_length(check_results)=0","array").count; abort "terminal rows without checks: #{bad}" unless bad.zero?; puts({terminal_without_checks:0}.to_json)' +} + +tmux kill-session -t municipal-qc-preform-review-v1 2>/dev/null || true +MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=1 bin/rails runner script/review_extracted_municipal_financial_statements.rb --release 2026-08-27 --provinces qc --retry-needs-review --batch-size 25 2>&1 | tee $output_root/qc-finite-review-before-fallback-v1-2026-08-30.jsonl +assert_terminal_checks + +canary_ids=$(MANIFEST="$manifest" bin/rails runner 'require "digest"; manifest=JSON.parse(File.read(ENV.fetch("MANIFEST"))); ids=manifest.fetch("document_ids").map { Integer(_1) }; expected={"position_net+position_surplus"=>16,"surplus_rollforward"=>9,"position-statement-not-found"=>7,"long-tail"=>18}; abort "manifest size" unless ids.length==50 && ids.uniq.length==50; abort "manifest checksum" unless Digest::SHA256.hexdigest(ids.join(","))=="ddf22a15cd12a32fcfab0ee303f6a0c28d45d4d5520fc410d3dd44ae26b8d834"; abort "manifest composition" unless manifest.fetch("records").map { _1.fetch("cluster") }.tally==expected; release=Warehouse::InstitutionRelease.find_by!(version:"2026-08-27"); candidates=Warehouse::FinancialStatementExtraction::CandidateSet.new(release:,provinces:["qc"]).each.to_a; filter=Warehouse::FinancialStatementExtraction::FailedCandidateFilter.new(release:,province:"qc",candidates:,parser_versions:["quebec-mamh-form-v1"]); report=filter.report; abort "target filter changed: #{report.slice(:aggregated_failure_count,:public_slot_count,:unmatched_failure_count,:reconciled).inspect}" unless report.fetch(:aggregated_failure_count)==719 && report.fetch(:public_slot_count)==719 && report.fetch(:unmatched_failure_count).zero? && report.fetch(:reconciled); eligible=candidates.select { filter.eligible?(_1) }.index_by(&:document_id); detailed=release.financial_statement_extractions.where(extractor_version:Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION); extractions=detailed.index_by { [_1.asset_sha256,_1.fiscal_year_end] }; manifest.fetch("records").each do |record|; candidate=eligible.fetch(Integer(record.fetch("document_id"))); extraction=extractions.fetch([candidate.asset_sha256,candidate.fiscal_year_end]); abort "canary identity drift" unless extraction.status=="failed" && extraction.llm_response_snapshot&.fetch("parser",nil)=="quebec-mamh-form-v1" && extraction.error_message==record.fetch("error"); end; active=detailed.where("institution_canonical_id LIKE ?","ca/qc/%").where(status:%w[pending extracting extracted]).count; abort "QC active detailed rows: #{active}" unless active.zero?; puts ids.join(",")' | tail -n 1) +echo "QC canary manifest revalidated: 50 targeted failures" + +tmux has-session -t municipal-qc-generic-canary-review-v1 2>/dev/null && { echo "QC canary reviewer exists"; exit 1; } +tmux new-session -d -s municipal-qc-generic-canary-review-v1 -c "$PWD" "set -o pipefail; PGHOST=127.0.0.1 PGPORT=55434 PGUSER=brendansamek MUNICIPAL_FINANCIAL_OCR_CACHE_ROOT=$ocr_cache_root MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=1 bin/rails runner script/review_extracted_municipal_financial_statements.rb --release 2026-08-27 --provinces qc --batch-size 25 --watch --idle-rounds 25920 2>&1 | tee $output_root/qc-generic-fallback-canary-review-watch-v1-2026-08-30.jsonl" +MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=2 bin/rails runner script/process_municipal_financial_statements.rb --release 2026-08-27 --province qc --rerun failed --failed-only --failed-parser quebec-mamh-form-v1 --document-ids "$canary_ids" 2>&1 | tee $output_root/qc-generic-fallback-canary-v1-2026-08-30.jsonl +drain_qc municipal-qc-generic-canary-review-v1 +tmux kill-session -t municipal-qc-generic-canary-review-v1 2>/dev/null || true +MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=1 bin/rails runner script/review_extracted_municipal_financial_statements.rb --release 2026-08-27 --provinces qc --retry-needs-review --batch-size 25 2>&1 | tee $output_root/qc-generic-fallback-canary-finite-review-v1-2026-08-30.jsonl +assert_terminal_checks + +MANIFEST="$manifest" bin/rails runner 'manifest=JSON.parse(File.read(ENV.fetch("MANIFEST"))); release=Warehouse::InstitutionRelease.find_by!(version:"2026-08-27"); version=Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION; records=manifest.fetch("records").map do |record|; extraction=release.financial_statement_extractions.find_by!(asset_sha256:record.fetch("asset_sha256"),fiscal_year_end:Date.iso8601(record.fetch("fiscal_year_end")),extractor_version:version); record.slice("document_id","document_canonical_id","institution_canonical_id","cluster").merge(status:extraction.status,extraction_id:extraction.id,reviewed_by:extraction.reviewed_by,reviewed_at:extraction.reviewed_at&.iso8601,fact_count:extraction.financial_statement_facts.count,line_item_count:extraction.financial_statement_line_items.count,verification:Warehouse::FinancialStatementExtraction.verification_checks(extraction.check_results),error:extraction.error_message); end; by_cluster=records.group_by { _1.fetch("cluster") }.transform_values { |rows| rows.map { _1.fetch(:status) }.tally }; puts({release:release.version,generated_at:Time.current.iso8601,gate:{required_approved:10,attempted:50},status_counts:records.map { _1.fetch(:status) }.tally,cluster_status_counts:by_cluster,records:}.to_json)' | tail -n 1 | tee "$canary_results" +gate=$(ruby -rjson -e 'payload=JSON.parse(File.read(ARGV.fetch(0))); counts=payload.fetch("status_counts"); puts [counts.fetch("approved",0),counts.fetch("needs_review",0),counts.fetch("failed",0)].join(":")' "$canary_results") +IFS=: read canary_approved canary_needs canary_failed <<< "$gate" +echo "QC canary gate approved=$canary_approved needs_review=$canary_needs failed=$canary_failed required_approved=10" +if test "$canary_approved" -lt 10; then + echo "QC canary gate failed; remainder intentionally not started; inspect saved per-cluster results" + exit 3 +fi + +tmux has-session -t municipal-qc-generic-remainder-review-v1 2>/dev/null && { echo "QC remainder reviewer exists"; exit 1; } +tmux new-session -d -s municipal-qc-generic-remainder-review-v1 -c "$PWD" "set -o pipefail; PGHOST=127.0.0.1 PGPORT=55434 PGUSER=brendansamek MUNICIPAL_FINANCIAL_OCR_CACHE_ROOT=$ocr_cache_root MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=1 bin/rails runner script/review_extracted_municipal_financial_statements.rb --release 2026-08-27 --provinces qc --batch-size 25 --watch --idle-rounds 25920 2>&1 | tee $output_root/qc-generic-fallback-remainder-review-watch-v1-2026-08-30.jsonl" +MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=2 bin/rails runner script/process_municipal_financial_statements.rb --release 2026-08-27 --province qc --rerun failed --failed-only --failed-parser quebec-mamh-form-v1 --exclude-document-ids "$canary_ids" 2>&1 | tee $output_root/qc-generic-fallback-remainder-v1-2026-08-30.jsonl +drain_qc municipal-qc-generic-remainder-review-v1 +tmux kill-session -t municipal-qc-generic-remainder-review-v1 2>/dev/null || true +MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=1 bin/rails runner script/review_extracted_municipal_financial_statements.rb --release 2026-08-27 --provinces qc --retry-needs-review --batch-size 25 2>&1 | tee $output_root/qc-generic-fallback-remainder-finite-review-v1-2026-08-30.jsonl +assert_terminal_checks +bin/rails runner 'release=Warehouse::InstitutionRelease.find_by!(version:"2026-08-27"); scope=release.financial_statement_extractions.where("institution_canonical_id LIKE ?","ca/qc/%").where(extractor_version:Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION); active=scope.where(status:%w[pending extracting extracted]).count; abort "QC undrained detailed rows: #{active}" unless active.zero?; puts({qc_detailed_pending_extracting_extracted:0}.to_json)' +bin/rails runner script/audit_municipal_financial_extraction_coverage.rb --release 2026-08-27 --provinces qc --output $output_root/qc-generic-fallback-final-coverage-v1-2026-08-30.json +echo "QC generic fallback canary and gated remainder complete" diff --git a/script/run_territory_financial_handoff.zsh b/script/run_territory_financial_handoff.zsh new file mode 100644 index 00000000..6706651d --- /dev/null +++ b/script/run_territory_financial_handoff.zsh @@ -0,0 +1,57 @@ +#!/bin/zsh + +set -e +set -o pipefail + +cd "${0:A:h}/.." + +output_root=/Volumes/floppy/york_factory/public_institutions/financial-extractions +ocr_cache_root=$output_root/ocr-cache-v1 +coordinator_log=$output_root/territories-after-nl-v1-2026-08-30.log +outputs=( + $output_root/nt-finite-review-before-retry-v1-2026-08-30.jsonl + $output_root/yt-finite-review-v1-2026-08-30.jsonl + $output_root/nt-generic-failed-v1-2026-08-30.jsonl + $output_root/nt-finite-review-after-retry-v1-2026-08-30.jsonl + $output_root/nt-final-coverage-v1-2026-08-30.json + $output_root/yt-final-coverage-v1-2026-08-30.json + $output_root/nu-final-coverage-v1-2026-08-30.json +) + +test ! -e "$coordinator_log" +for output_path in $outputs; do + test ! -e "$output_path" || { echo "refusing existing output: $output_path"; exit 1; } +done +exec > >(tee "$coordinator_log") 2>&1 +export PGHOST=127.0.0.1 PGPORT=55434 PGUSER=brendansamek MUNICIPAL_FINANCIAL_OCR_CACHE_ROOT=$ocr_cache_root + +polls=0 +while true; do + dead=$(tmux display-message -p -t municipal-nl-retry-review-after-v2:0 '#{pane_dead}' 2>/dev/null) || { + echo "NL coordinator disappeared" + exit 1 + } + if test "$dead" = "1"; then + exit_status=$(tmux display-message -p -t municipal-nl-retry-review-after-v2:0 '#{pane_dead_status}') + test "$exit_status" = "0" || { echo "NL coordinator failed with status $exit_status"; exit 1; } + break + fi + polls=$((polls + 1)) + test "$polls" -lt 8640 || { echo "NL coordinator wait timed out"; exit 1; } + sleep 30 +done + +bin/rails runner 'release=Warehouse::InstitutionRelease.find_by!(version:"2026-08-27"); version=Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION; expected={"nt"=>{"approved"=>17,"needs_review"=>9,"failed"=>2},"yt"=>{"approved"=>22,"needs_review"=>18},"nu"=>{"approved"=>15}}; expected.each do |province,statuses|; totals=Warehouse::FinancialStatementExtraction::CoverageAudit.new(release:,provinces:[province]).payload.fetch(:totals); abort "#{province.upcase} preflight changed: #{totals.fetch(:status_counts).inspect}" unless totals.fetch(:status_counts)==statuses; abort "#{province.upcase} checks/provenance failed" unless totals.fetch(:approved_without_checks).zero? && totals.fetch(:approved_without_deterministic_reviewer).zero?; active=release.financial_statement_extractions.where("institution_canonical_id LIKE ?","ca/#{province}/%").where(extractor_version:version,status:%w[pending extracting extracted]).count; abort "#{province.upcase} active detailed rows: #{active}" unless active.zero?; end; puts({territory_preflight:"pass",statuses:expected}.to_json)' + +MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=1 bin/rails runner script/review_extracted_municipal_financial_statements.rb --release 2026-08-27 --provinces nt --retry-needs-review --batch-size 25 2>&1 | tee $output_root/nt-finite-review-before-retry-v1-2026-08-30.jsonl +MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=1 bin/rails runner script/review_extracted_municipal_financial_statements.rb --release 2026-08-27 --provinces yt --retry-needs-review --batch-size 25 2>&1 | tee $output_root/yt-finite-review-v1-2026-08-30.jsonl + +bin/rails runner 'release=Warehouse::InstitutionRelease.find_by!(version:"2026-08-27"); candidates=Warehouse::FinancialStatementExtraction::CandidateSet.new(release:,provinces:["nt"]).each.to_a; report=Warehouse::FinancialStatementExtraction::FailedCandidateFilter.new(release:,province:"nt",candidates:).report; covered=report.fetch(:approved_elsewhere_excluded_count)+report.fetch(:review_pending_elsewhere_excluded_count); abort "NT failed filter changed: #{report.slice(:aggregated_failure_count,:public_slot_count,:approved_elsewhere_excluded_count,:review_pending_elsewhere_excluded_count,:duplicate_slot_excluded_count,:unmatched_failure_count,:reconciled).inspect}" unless report.fetch(:aggregated_failure_count)==2 && report.fetch(:public_slot_count)+covered==2 && report.fetch(:duplicate_slot_excluded_count).zero? && report.fetch(:unmatched_failure_count).zero? && report.fetch(:reconciled); puts({nt_failed_filter:report.except(:approved_elsewhere_excluded,:review_pending_elsewhere_excluded,:duplicate_slot_excluded,:unmatched_failures)}.to_json)' +bin/rails runner script/process_municipal_financial_statements.rb --release 2026-08-27 --province nt --rerun failed --failed-only 2>&1 | tee $output_root/nt-generic-failed-v1-2026-08-30.jsonl +MUNICIPAL_FINANCIAL_OCR_CONCURRENCY=1 bin/rails runner script/review_extracted_municipal_financial_statements.rb --release 2026-08-27 --provinces nt --retry-needs-review --batch-size 25 2>&1 | tee $output_root/nt-finite-review-after-retry-v1-2026-08-30.jsonl + +bin/rails runner 'release=Warehouse::InstitutionRelease.find_by!(version:"2026-08-27"); version=Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION; %w[nt yt].each do |province|; scope=release.financial_statement_extractions.where("institution_canonical_id LIKE ?","ca/#{province}/%").where(extractor_version:version); active=scope.where(status:%w[pending extracting extracted]).count; abort "#{province.upcase} undrained detailed rows: #{active}" unless active.zero?; end; nu=Warehouse::FinancialStatementExtraction::CoverageAudit.new(release:,provinces:["nu"]).payload.fetch(:totals); abort "NU changed: #{nu.fetch(:status_counts).inspect}" unless nu.fetch(:preferred_asset_count)==15 && nu.fetch(:published_institution_year_count)==15 && nu.fetch(:status_counts)=={"approved"=>15} && nu.fetch(:approved_without_checks).zero? && nu.fetch(:approved_without_deterministic_reviewer).zero?; terminal=%w[extracted needs_review approved rejected failed]; bad=Warehouse::FinancialStatementExtraction.where(status:terminal).where("jsonb_typeof(check_results) <> ? OR jsonb_array_length(check_results)=0","array").count; abort "terminal rows without checks: #{bad}" unless bad.zero?; puts({territories_drained:true,nu_approved:15,terminal_without_checks:0}.to_json)' +bin/rails runner script/audit_municipal_financial_extraction_coverage.rb --release 2026-08-27 --provinces nt --output $output_root/nt-final-coverage-v1-2026-08-30.json +bin/rails runner script/audit_municipal_financial_extraction_coverage.rb --release 2026-08-27 --provinces yt --output $output_root/yt-final-coverage-v1-2026-08-30.json +bin/rails runner script/audit_municipal_financial_extraction_coverage.rb --release 2026-08-27 --provinces nu --output $output_root/nu-final-coverage-v1-2026-08-30.json +echo "Territory extraction, review, and per-record audits complete" diff --git a/script/sanitize_municipal_report_batch.rb b/script/sanitize_municipal_report_batch.rb index a712ce37..63de38cf 100644 --- a/script/sanitize_municipal_report_batch.rb +++ b/script/sanitize_municipal_report_batch.rb @@ -13,7 +13,7 @@ class SanitizeMunicipalReportBatch DEFAULT_ASSET_ROOT = Pathname("/Volumes/floppy/york_factory/public_institutions/assets") YEAR_PATTERN = /(? 2, "pass" => 1, "skip" => 1, "fail" => 0 }, statement.dig("verification", "summary")) + assert_equal verification_checks.map { _1.stringify_keys }, statement.dig("verification", "checks") + end + + test "unknown verification statuses count as failures" do + @extraction.update_column(:check_results, verification_checks + [ + { id: "future_check", status: "unknown", detail: "new status" } + ]) + + get "/api/v1/warehouse/municipal_financial_statements/on/example-town/2025" + + assert_equal 1, JSON.parse(response.body).dig("statements", 0, "verification", "summary", "fail") + end + + test "does not publish a Sankey that would contradict headline totals" do + @extraction.financial_statement_line_items.create!( + flow: "revenue", category: "Adjustments", label: "Consolidation adjustment", value: -20_000, + raw_text: "(20,000)", scale: 1, source_page: 20, column_year: "2025", + position: 1, extraction_confidence: 0.98 + ) + + get "/api/v1/warehouse/municipal_financial_statements/on/example-town/2025" + + statement = JSON.parse(response.body).fetch("statements").sole + assert_nil statement.fetch("sankey") + assert_equal(-20_000.0, + statement.fetch("line_items").find { _1["label"] == "Consolidation adjustment" }.fetch("value")) + end + + test "repositions signed adjustments into a nonnegative Sankey without changing reported rows" do + @extraction.financial_statement_line_items.find_by!(flow: "revenue").update_column(:value, 12_345_778) + @extraction.financial_statement_line_items.find_by!(flow: "expense").update_column(:value, 10_000_040) + @extraction.financial_statement_line_items.create!( + flow: "revenue", category: "Adjustments", label: "Loss on disposal", value: -100, + raw_text: "(100)", scale: 1, source_page: 20, column_year: "2025", + position: 1, extraction_confidence: 0.98 + ) + @extraction.financial_statement_line_items.create!( + flow: "expense", category: "Adjustments", label: "Expense recovery", value: -40, + raw_text: "(40)", scale: 1, source_page: 21, column_year: "2025", + position: 1, extraction_confidence: 0.98 + ) + @extraction.financial_statement_line_items.create!( + flow: "revenue", category: "Adjustments", label: "Nil adjustment", value: 0, + raw_text: "-", scale: 1, source_page: 20, column_year: "2025", + position: 2, extraction_confidence: 0.98 + ) + + get "/api/v1/warehouse/municipal_financial_statements/on/example-town/2025" + + statement = JSON.parse(response.body).fetch("statements").sole + sankey = statement.fetch("sankey") + revenue_leaves = sankey.dig("revenue_data", "children").flat_map { _1.fetch("children") } + spending_leaves = sankey.dig("spending_data", "children").flat_map { _1.fetch("children") } + assert_equal "Inflows", sankey.dig("revenue_data", "name") + assert_equal "Outflows", sankey.dig("spending_data", "name") + assert_equal 12_345_678.0, sankey.fetch("revenue") + assert_equal 10_000_000.0, sankey.fetch("spending") + assert_equal 12_345_818.0, sankey.fetch("total") + assert_equal 2_345_678.0, revenue_leaves.sum { _1.fetch("amount") } - + spending_leaves.sum { _1.fetch("amount") } + assert spending_leaves.any? { _1.fetch("id").start_with?("expense-revenue-") && _1.fetch("name") == "Loss on disposal" } + assert revenue_leaves.any? { _1.fetch("id").start_with?("revenue-expense-") && _1.fetch("name") == "Expense recovery" } + refute (revenue_leaves + spending_leaves).any? { _1.fetch("name") == "Nil adjustment" } + assert_equal(-100.0, statement.fetch("line_items").find { _1["label"] == "Loss on disposal" }.fetch("value")) + assert_equal(-40.0, statement.fetch("line_items").find { _1["label"] == "Expense recovery" }.fetch("value")) + assert_equal 0.0, statement.fetch("line_items").find { _1["label"] == "Nil adjustment" }.fetch("value") + end + + test "publishes a positive Sankey using an independently verified adjustment basis" do + @extraction.financial_statement_line_items.create!( + flow: "revenue", category: "Contributions", label: "Capital contributions", value: 100, + raw_text: "100", scale: 1, source_page: 20, column_year: "2025", + position: 1, extraction_confidence: 0.98 + ) + @extraction.update_column(:check_results, verification_checks + [ + { id: "line_sum:revenue", status: "pass", detail: "verified adjustment basis" } + ]) + + get "/api/v1/warehouse/municipal_financial_statements/on/example-town/2025" + + statement = JSON.parse(response.body).fetch("statements").sole + assert_equal 12_345_778.0, statement.dig("sankey", "revenue") + assert_equal "Capital contributions", + statement.dig("sankey", "revenue_data", "children", 1, "children", 0, "name") + end + + test "newest release wins when a fiscal year has multiple approved extractions" do + newer_release = Warehouse::InstitutionRelease.create!( + version: "2026-08-29", effective_on: Date.new(2026, 8, 29), schema_version: "1.0", + published_at: Time.utc(2026, 8, 29), geography_vintage: 2021, attribution: "Test" + ) + source = Warehouse::InstitutionSource.create!( + institution_release: newer_release, canonical_id: "ca/sources/api-test", + publisher_name: "Test", title_en: "Test", url: "https://example.test/source", + retrieved_at: newer_release.published_at, languages: [ "en" ] + ) + institution = Warehouse::Institution.create!( + institution_release: newer_release, institution_source: source, + canonical_id: @institution.canonical_id, name_en: "Example Town Updated", + institution_type: "government", government_level: "municipal", status: "active" + ) + document = Warehouse::InstitutionDocument.create!( + institution_release: newer_release, institution:, institution_source: source, + canonical_id: "ca/on/example-town/documents/financial-statements/2025/general", + document_type: "financial-statements", document_variant: "general" + ) + asset = Warehouse::InstitutionDocumentAsset.create!( + institution_release: newer_release, institution_document: document, content_sha256: "e" * 64, + asset_role: "final", preferred: true, download_url: "https://example.test/newer.pdf", + retrieved_at: newer_release.published_at, archive_path: "sha256/ee/#{'e' * 64}.pdf", + mime_type: "application/pdf", byte_size: 100, rights_status: "metadata_only" + ) + extraction = Warehouse::FinancialStatementExtraction.create!( + institution_release: newer_release, institution_canonical_id: institution.canonical_id, + document_canonical_id: document.canonical_id, asset_sha256: asset.content_sha256, + fiscal_year_end: Date.new(2025, 12, 31), + extractor_version: Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION, + status: "extracted", check_results: verification_checks + ) + extraction.financial_statement_facts.create!( + concept: "total_revenue", value: 99_000_000, raw_text: "99,000,000", + raw_label: "Total revenue", scale: 1, statement: "operations", source_page: 9, + column_year: "2025", extraction_confidence: 0.99 + ) + extraction.approve!(reviewer: "reviewer") + + get "/api/v1/warehouse/municipal_financial_statements/ontario/example-town/2025" + + assert_response :success + payload = JSON.parse(response.body) + assert_equal "Example Town Updated", payload.fetch("name") + assert_equal 99_000_000.0, payload.dig("statements", 0, "facts", 0, "value") + assert_equal "https://example.test/newer.pdf", payload.dig("statements", 0, "source", "download_url") + end + + test "show does not expose unapproved extractions" do + @extraction.update!(status: "rejected") + + get "/api/v1/warehouse/municipal_financial_statements/on/example-town/2025" + + assert_response :not_found + end + + test "nested regional-government ids use stable route slugs" do + institution = Warehouse::Institution.create!( + institution_release: @release, institution_source: @source, + canonical_id: "ca/bc/regional/example", name_en: "Example Regional District", + institution_type: "government", government_level: "regional", status: "active" + ) + document = Warehouse::InstitutionDocument.create!( + institution_release: @release, institution:, institution_source: @source, + canonical_id: "ca/bc/regional/example/documents/financial-statements/2025/general", + document_type: "financial-statements", document_variant: "general" + ) + asset = Warehouse::InstitutionDocumentAsset.create!( + institution_release: @release, institution_document: document, content_sha256: "b" * 64, + asset_role: "final", preferred: true, download_url: "https://example.test/regional.pdf", + retrieved_at: @release.published_at, archive_path: "sha256/bb/#{'b' * 64}.pdf", + mime_type: "application/pdf", byte_size: 100, rights_status: "metadata_only" + ) + extraction = Warehouse::FinancialStatementExtraction.create!( + institution_release: @release, institution_canonical_id: institution.canonical_id, + document_canonical_id: document.canonical_id, asset_sha256: asset.content_sha256, + fiscal_year_end: Date.new(2025, 12, 31), + extractor_version: Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION, + status: "extracted", check_results: verification_checks + ) + extraction.approve!(reviewer: "reviewer") + + get "/api/v1/warehouse/municipal_financial_statements/bc/regional--example/2025" + + assert_response :success + payload = JSON.parse(response.body) + assert_equal "regional--example", payload.fetch("slug") + assert_equal "ca/bc/regional/example", payload.fetch("canonical_id") + end + + private + + def create_approved_municipality(canonical_id, name, sha) + institution = Warehouse::Institution.create!( + institution_release: @release, institution_source: @source, canonical_id:, name_en: name, + institution_type: "government", government_level: "municipal", status: "active" + ) + document = Warehouse::InstitutionDocument.create!( + institution_release: @release, institution:, institution_source: @source, + canonical_id: "#{canonical_id}/documents/financial-statements/2025/general", + document_type: "financial-statements", document_variant: "general" + ) + asset = Warehouse::InstitutionDocumentAsset.create!( + institution_release: @release, institution_document: document, content_sha256: sha, + asset_role: "final", preferred: true, download_url: "https://example.test/zeta.pdf", + retrieved_at: @release.published_at, archive_path: "sha256/#{sha.first(2)}/#{sha}.pdf", + mime_type: "application/pdf", byte_size: 100, rights_status: "metadata_only" + ) + extraction = Warehouse::FinancialStatementExtraction.create!( + institution_release: @release, institution_canonical_id: canonical_id, + document_canonical_id: document.canonical_id, asset_sha256: asset.content_sha256, + fiscal_year_end: Date.new(2025, 12, 31), + extractor_version: Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION, + status: "extracted", check_results: verification_checks + ) + extraction.approve!(reviewer: "reviewer") + end + + def verification_checks + [ + { id: "source_identity", status: "pass", detail: "source hash matches" }, + { id: "population_context", status: "skip", detail: "not required for approval" } + ] + end +end diff --git a/test/jobs/warehouse/extract_municipal_financial_statements_job_test.rb b/test/jobs/warehouse/extract_municipal_financial_statements_job_test.rb new file mode 100644 index 00000000..5330b07b --- /dev/null +++ b/test/jobs/warehouse/extract_municipal_financial_statements_job_test.rb @@ -0,0 +1,91 @@ +require "test_helper" +require "active_job/continuation/test_helper" + +class TestMunicipalCandidateSet + def initialize(candidates) = @candidates = candidates + def asset_root = Pathname(Dir.tmpdir) + + def each(start: nil) + return enum_for(__method__, start:) unless block_given? + + @candidates.select { start.nil? || _1.document_id >= start }.each { yield _1 } + end +end + +class TestExtractMunicipalFinancialStatementsJob < Warehouse::ExtractMunicipalFinancialStatementsJob + cattr_accessor :candidates, default: [] + cattr_accessor :results, default: [] + cattr_accessor :processed_ids, default: [] + + private + + def candidate_set(**) = TestMunicipalCandidateSet.new(self.class.candidates) + + def processor(**) + lambda do |candidate| + self.class.processed_ids << candidate.document_id + self.class.results.fetch(candidate.document_id) + end + end +end + +class Warehouse::ExtractMunicipalFinancialStatementsJobTest < ActiveJob::TestCase + include ActiveJob::Continuation::TestHelper + + setup do + @release = Warehouse::InstitutionRelease.create!( + version: "2026-08-31", effective_on: Date.new(2026, 8, 31), schema_version: "1.0", + published_at: Time.utc(2026, 8, 31), geography_vintage: 2021, attribution: "Test" + ) + TestExtractMunicipalFinancialStatementsJob.candidates = [ candidate(10), candidate(20), candidate(30) ] + TestExtractMunicipalFinancialStatementsJob.results = [ 10, 20, 30 ].to_h do |id| + [ id, outcome("extracted") ] + end + TestExtractMunicipalFinancialStatementsJob.processed_ids = [] + end + + test "resumes from the next stable document id" do + TestExtractMunicipalFinancialStatementsJob.perform_later(@release.version, province: "on") + interrupt_job_during_step( + TestExtractMunicipalFinancialStatementsJob, :extract_statements, cursor: 11 + ) { perform_enqueued_jobs } + + assert_equal [ 10 ], TestExtractMunicipalFinancialStatementsJob.processed_ids + perform_enqueued_jobs + assert_equal [ 10, 20, 30 ], TestExtractMunicipalFinancialStatementsJob.processed_ids + end + + test "one failed statement does not halt the province" do + TestExtractMunicipalFinancialStatementsJob.results[20] = outcome("failed") + + TestExtractMunicipalFinancialStatementsJob.perform_now(@release.version, province: "on") + + assert_equal [ 10, 20, 30 ], TestExtractMunicipalFinancialStatementsJob.processed_ids + end + + test "opens the circuit after a sustained failure window" do + TestExtractMunicipalFinancialStatementsJob.candidates = (1..20).map { candidate(_1) } + TestExtractMunicipalFinancialStatementsJob.results = (1..20).to_h { [ _1, outcome("failed") ] } + + assert_raises(Warehouse::ExtractMunicipalFinancialStatementsJob::CircuitOpen) do + TestExtractMunicipalFinancialStatementsJob.perform_now(@release.version, province: "on") + end + end + + private + + def candidate(id) + Warehouse::FinancialStatementExtraction::CandidateSet::Candidate.new( + document_id: id, institution_canonical_id: "ca/on/example", + institution_name: "Example", document_canonical_id: "ca/on/example/documents/financial-statements/2025/general", + asset_sha256: "a" * 64, fiscal_year_end: Date.new(2025, 12, 31), + pdf_path: Pathname("unused.pdf"), population: 1 + ) + end + + def outcome(status) + Warehouse::FinancialStatementExtraction::Processor::Outcome.new( + status:, stage: "test", extraction_id: nil, error: status == "failed" ? "test" : nil + ) + end +end diff --git a/test/models/warehouse/census_profile_importer_test.rb b/test/models/warehouse/census_profile_importer_test.rb new file mode 100644 index 00000000..dd3f23c3 --- /dev/null +++ b/test/models/warehouse/census_profile_importer_test.rb @@ -0,0 +1,32 @@ +require "test_helper" + +class Warehouse::CensusProfileImporterTest < ActiveSupport::TestCase + test "imports versioned 2021 CSD population rows idempotently" do + csv = <<~CSV + CENSUS_YEAR,DGUID,ALT_GEO_CODE,GEO_LEVEL,GEO_NAME,TNR_SF,TNR_LF,DATA_QUALITY_FLAG,CHARACTERISTIC_ID,CHARACTERISTIC_NAME,CHARACTERISTIC_NOTE,C1_COUNT_TOTAL + 2021,2021A00054811061,4811061,Census subdivision,Edmonton,0,0,0,1,"Population, 2021",1,1010899 + 2021,2021A00054811061,4811061,Census subdivision,Edmonton,0,0,0,2,"Population, 2016",1,932546 + 2021,2021A00054811061,4811061,Census subdivision,Edmonton,0,0,0,6,"Population density per square kilometre",1,1320.4 + 2021,2021A00054811061,4811061,Census subdivision,Edmonton,0,0,0,7,"Land area in square kilometres",1,765.61 + CSV + + Dir.mktmpdir do |dir| + source = Pathname(dir).join(Warehouse::CensusProfileImporter::CSV_ENTRY) + source.write(csv) + archive = Pathname(dir).join("profile.zip") + system("zip", "-q", archive.to_s, source.basename.to_s, chdir: dir) + sha = Digest::SHA256.file(archive).hexdigest + importer = Warehouse::CensusProfileImporter.new( + zip_path: archive, expected_sha256: sha, retrieved_at: Time.utc(2026, 8, 29) + ) + + 2.times { importer.import! } + + profile = Warehouse::CensusProfile.sole + assert_equal 1_010_899, profile.population + assert_equal "4811061", profile.geo_uid + assert_equal BigDecimal("765.61"), profile.area_sq_km + assert_equal BigDecimal("1320.4"), profile.population_density_per_sq_km + end + end +end diff --git a/test/models/warehouse/financial_statement_extraction/candidate_set_test.rb b/test/models/warehouse/financial_statement_extraction/candidate_set_test.rb new file mode 100644 index 00000000..72e8eef9 --- /dev/null +++ b/test/models/warehouse/financial_statement_extraction/candidate_set_test.rb @@ -0,0 +1,79 @@ +require "test_helper" + +class Warehouse::FinancialStatementExtraction::CandidateSetTest < ActiveSupport::TestCase + setup do + @release = Warehouse::InstitutionRelease.create!( + version: "2026-08-30", effective_on: Date.new(2026, 8, 30), schema_version: "1.0", + published_at: Time.utc(2026, 8, 30), geography_vintage: 2021, attribution: "Test" + ) + @source = Warehouse::InstitutionSource.create!( + institution_release: @release, canonical_id: "ca/sources/candidate-test", + publisher_name: "Test", title_en: "Test", url: "https://example.test/source", + retrieved_at: @release.published_at, languages: [ "en" ] + ) + @directory = Pathname(Dir.mktmpdir) + end + + teardown { FileUtils.remove_entry(@directory) } + + test "selects every preferred local-government PDF and derives missing fiscal dates" do + municipal = create_institution("ca/on/example", "municipal") + regional = create_institution("ca/bc/example-regional-district", "regional") + create_document(municipal, year: 2024, fiscal_period_end: nil, sha: "a" * 64) + create_document(regional, year: 2023, fiscal_period_end: Date.new(2023, 12, 31), sha: "b" * 64) + + candidates = Warehouse::FinancialStatementExtraction::CandidateSet.new( + release: @release, asset_root: @directory + ).each.to_a + + assert_equal 2, candidates.length + assert_equal [ 2023, 2024 ], candidates.map { _1.fiscal_year_end.year }.sort + assert_equal %w[ca/bc/example-regional-district ca/on/example], + candidates.map(&:institution_canonical_id).sort + end + + test "filters by province and year and audits the archived hash" do + institution = create_institution("ca/on/example", "municipal") + create_document(institution, year: 2024, fiscal_period_end: nil, sha: Digest::SHA256.hexdigest("pdf"), contents: "pdf") + create_document(institution, year: 2023, fiscal_period_end: nil, sha: Digest::SHA256.hexdigest("older"), contents: "older") + + set = Warehouse::FinancialStatementExtraction::CandidateSet.new( + release: @release, provinces: [ "on" ], years: [ 2024 ], asset_root: @directory + ) + audit = set.audit(verify_hashes: true) + + assert_equal 1, set.count + assert_equal 1, audit.fetch(:candidates) + assert_empty audit.fetch(:missing_files) + assert_empty audit.fetch(:size_mismatches) + assert_empty audit.fetch(:hash_mismatches) + end + + private + + def create_institution(canonical_id, government_level) + Warehouse::Institution.create!( + institution_release: @release, institution_source: @source, canonical_id:, + name_en: canonical_id.split("/").last.titleize, institution_type: "government", + government_level:, status: "active" + ) + end + + def create_document(institution, year:, fiscal_period_end:, sha:, contents: "x") + document = Warehouse::InstitutionDocument.create!( + institution_release: @release, institution:, institution_source: @source, + canonical_id: "#{institution.canonical_id}/documents/financial-statements/#{year}/general", + document_type: "financial-statements", document_variant: "general", fiscal_period_end: + ) + relative = Pathname("sha256/#{sha.first(2)}/#{sha}.pdf") + path = @directory.join(relative) + path.dirname.mkpath + path.write(contents) + Warehouse::InstitutionDocumentAsset.create!( + institution_release: @release, institution_document: document, content_sha256: sha, + asset_role: "final", preferred: true, download_url: "https://example.test/#{year}.pdf", + retrieved_at: @release.published_at, archive_path: relative.to_s, + mime_type: "application/pdf", byte_size: path.size, rights_status: "metadata_only" + ) + end +end diff --git a/test/models/warehouse/financial_statement_extraction/candidate_window_test.rb b/test/models/warehouse/financial_statement_extraction/candidate_window_test.rb new file mode 100644 index 00000000..bbb9d2d2 --- /dev/null +++ b/test/models/warehouse/financial_statement_extraction/candidate_window_test.rb @@ -0,0 +1,29 @@ +require "test_helper" + +class Warehouse::FinancialStatementExtraction::CandidateWindowTest < ActiveSupport::TestCase + Candidate = Data.define(:document_id) + + test "includes the document before the boundary and stops at the boundary" do + window = Warehouse::FinancialStatementExtraction::CandidateWindow.new( + start: 9251, stop_before: 9252 + ) + + refute window.before_start?(Candidate.new(document_id: 9251)) + refute window.at_or_after_stop?(Candidate.new(document_id: 9251)) + assert window.at_or_after_stop?(Candidate.new(document_id: 9252)) + end + + test "supports explicit canary inclusion and remainder exclusion" do + canary = Warehouse::FinancialStatementExtraction::CandidateWindow.new( + document_ids: [ 10, 12 ] + ) + remainder = Warehouse::FinancialStatementExtraction::CandidateWindow.new( + excluded_document_ids: [ 10, 12 ] + ) + + assert canary.selected?(Candidate.new(document_id: 10)) + refute canary.selected?(Candidate.new(document_id: 11)) + assert remainder.excluded?(Candidate.new(document_id: 12)) + refute remainder.excluded?(Candidate.new(document_id: 13)) + end +end diff --git a/test/models/warehouse/financial_statement_extraction/detailed_pipeline_test.rb b/test/models/warehouse/financial_statement_extraction/detailed_pipeline_test.rb new file mode 100644 index 00000000..f760f4e1 --- /dev/null +++ b/test/models/warehouse/financial_statement_extraction/detailed_pipeline_test.rb @@ -0,0 +1,353 @@ +require "test_helper" + +class Warehouse::FinancialStatementExtraction::DetailedPipelineTest < ActiveSupport::TestCase + test "normalizes detailed line items and accepts reconciled leaf sums" do + facts = [ + fact("total_financial_assets", "100,000", 100_000_000, "financial_position"), + fact("total_liabilities", "60,000", 60_000_000, "financial_position"), + fact("net_financial_assets", "40,000", 40_000_000, "financial_position"), + fact("total_non_financial_assets", "160,000", 160_000_000, "financial_position"), + fact("accumulated_surplus", "200,000", 200_000_000, "financial_position"), + fact("total_revenue", "80,000", 80_000_000, "operations"), + fact("total_expenses", "70,000", 70_000_000, "operations"), + fact("annual_surplus", "10,000", 10_000_000, "operations") + ] + page_text = facts.map { |row| "#{row[:raw_label]} #{row[:raw_text]}" }.join(" ") + + " Revenue expense schedule Property taxes 80,000 Operations 70,000" + locator_result = Warehouse::FinancialStatementExtraction::PageLocator::Result.new( + page_count: 1, page_texts: { 1 => page_text }, position_page: 1, + operations_page: 1, candidate_pages: [ 1 ], ocr_pages: [] + ) + headline = Warehouse::FinancialStatementExtraction::Pipeline::Result.new( + status: "extracted", facts:, checks: [ { id: "source_identity", status: "pass", detail: "ok" } ], + prompt: "headline", response: { + "remeasurement_present" => false, "operations_adjustment_present" => false, + "rollforward_adjustment_present" => false + }, locator_result:, language: "en", statement_basis: "consolidated" + ) + headline_pipeline = Struct.new(:result) { def run = result }.new(headline) + page_locator = Struct.new(:result, :excerpt_calls) do + def with_excerpt(pages) + excerpt_calls << pages + yield Pathname("detail-#{pages.join('-')}.pdf") + end + end.new(locator_result, []) + attachments = [] + responses = { + "revenue" => { "flow" => "revenue", "category" => "Taxes", "label" => "Property taxes", + "raw_text" => "80,000", "scale" => 1_000, "excerpt_page" => 1, + "column_year" => "2025", "confidence" => 0.99 }, + "expense" => { "flow" => "expense", "category" => "Services", "label" => "Operations", + "raw_text" => "70,000", "scale" => 1_000, "excerpt_page" => 1, + "column_year" => "2025", "confidence" => 0.99 }, + "empty" => { "flow" => "expense", "category" => "Services", "label" => "Unused program", + "raw_text" => '"', "scale" => 1_000, "excerpt_page" => 1, + "column_year" => "2025", "confidence" => 0.99 } + } + active_flow_calls = 0 + maximum_flow_calls = 0 + flow_call_mutex = Mutex.new + pipeline = Warehouse::FinancialStatementExtraction::DetailedPipeline.new( + pdf_path: "unused.pdf", institution_canonical_id: "ca/on/example", + institution_name: "Example", document_canonical_id: "ca/on/example/documents/financial-statements/2025/general", + asset_sha256: "a" * 64, fiscal_year_end: Date.new(2025, 12, 31), + headline_pipeline:, page_locator:, llm_client: ->(prompt:, pdf_path:) do + flow_call_mutex.synchronize do + active_flow_calls += 1 + maximum_flow_calls = [ maximum_flow_calls, active_flow_calls ].max + end + attachments << pdf_path + flow = prompt.include?("REQUESTED FLOW: revenue") ? "revenue" : "expense" + items = [ responses.fetch(flow) ] + items << responses.fetch("empty") if flow == "expense" + { "fiscal_year" => 2025, "line_items" => items } + ensure + flow_call_mutex.synchronize { active_flow_calls -= 1 } + end + ) + + result = pipeline.run + + assert_equal "extracted", result.status + assert_equal [ "revenue", "expense" ], result.line_items.map { |item| item.fetch(:flow) } + assert_equal BigDecimal("80000000"), result.line_items.first.fetch(:value) + assert_equal [ [ 1 ], [ 1 ] ], page_locator.excerpt_calls + assert_equal [ "detail-1.pdf", "detail-1.pdf" ], attachments + assert_equal 1, maximum_flow_calls + + headline_pipeline.result = headline.with(status: "needs_review") + assert_equal "needs_review", pipeline.run.status + end + + test "adds flow-specific schedule pages to the primary operations page" do + locator = Warehouse::FinancialStatementExtraction::PageLocator::Result.new( + page_count: 4, + page_texts: { + 1 => "Statement of Operations 2025 Revenue 80,000 Expenses 70,000", + 2 => "Schedule A - Revenue 2025 Taxes 50,000 Transfers 30,000", + 3 => "Schedule B - Expenses 2025 Wages 40,000 Supplies 30,000", + 4 => "Notes 2025 Revenue recognition 80,000" + }, + position_page: 1, operations_page: 1, candidate_pages: [ 1 ], ocr_pages: [] + ) + pipeline = Warehouse::FinancialStatementExtraction::DetailedPipeline.new( + pdf_path: "unused.pdf", institution_canonical_id: "ca/on/example", + institution_name: "Example", + document_canonical_id: "ca/on/example/documents/financial-statements/2025/general", + asset_sha256: "a" * 64, fiscal_year_end: Date.new(2025, 12, 31) + ) + + assert_equal [ 1, 2 ], pipeline.send(:select_detail_pages, locator, "revenue") + assert_equal [ 1, 3 ], pipeline.send(:select_detail_pages, locator, "expense") + end + + test "caps detail extraction at the operations page plus three schedules" do + page_texts = { 1 => "Statement of Operations 2025 Revenue 80,000 Expenses 70,000" } + 5.times do |index| + page_texts[index + 2] = "Schedule #{index + 1} - Expenses 2025 Wages #{40_000 + index},000 Supplies 30,000" + end + locator = Warehouse::FinancialStatementExtraction::PageLocator::Result.new( + page_count: 6, page_texts:, position_page: 1, operations_page: 1, + candidate_pages: [ 1 ], ocr_pages: [] + ) + pipeline = Warehouse::FinancialStatementExtraction::DetailedPipeline.new( + pdf_path: "unused.pdf", institution_canonical_id: "ca/on/example", + institution_name: "Example", + document_canonical_id: "ca/on/example/documents/financial-statements/2025/general", + asset_sha256: "a" * 64, fiscal_year_end: Date.new(2025, 12, 31) + ) + + assert_equal 4, pipeline.send(:select_detail_pages, locator, "expense").length + end + + test "sorts detail pages to match the attached excerpt order" do + locator = Warehouse::FinancialStatementExtraction::PageLocator::Result.new( + page_count: 4, + page_texts: { + 1 => "Schedule A - Expenses 2025 Wages 40,000 Supplies 30,000", + 3 => "Statement of Operations 2025 Revenue 80,000 Expenses 70,000" + }, + position_page: 3, operations_page: 3, candidate_pages: [ 3 ], ocr_pages: [] + ) + pipeline = Warehouse::FinancialStatementExtraction::DetailedPipeline.new( + pdf_path: "unused.pdf", institution_canonical_id: "ca/on/example", + institution_name: "Example", + document_canonical_id: "ca/on/example/documents/financial-statements/2025/general", + asset_sha256: "a" * 64, fiscal_year_end: Date.new(2025, 12, 31) + ) + + assert_equal [ 1, 3 ], pipeline.send(:select_detail_pages, locator, "expense") + end + + test "uses only the primary operations page when it already prints detailed flow rows" do + locator = Warehouse::FinancialStatementExtraction::PageLocator::Result.new( + page_count: 3, + page_texts: { + 1 => <<~TEXT, + Statement of Operations + Revenue + Property taxes 1,000 900 + Government transfers 2,000 1,800 + Expenses + General government 1,200 1,100 + Transportation 1,300 1,200 + Total expenses 2,500 2,300 + TEXT + 2 => "Schedule A - Revenue 2025 Taxes 1,000 Transfers 2,000", + 3 => "Schedule B - Expenses 2025 Wages 1,200 Supplies 1,300" + }, + position_page: 1, operations_page: 1, candidate_pages: [ 1 ], ocr_pages: [] + ) + pipeline = Warehouse::FinancialStatementExtraction::DetailedPipeline.new( + pdf_path: "unused.pdf", institution_canonical_id: "ca/on/example", + institution_name: "Example", + document_canonical_id: "ca/on/example/documents/financial-statements/2025/general", + asset_sha256: "a" * 64, fiscal_year_end: Date.new(2025, 12, 31) + ) + + assert_equal [ 1 ], pipeline.send(:select_detail_pages, locator, "revenue") + assert_equal [ 1 ], pipeline.send(:select_detail_pages, locator, "expense") + prompt = pipeline.send(:build_prompt, [ 1 ], locator.page_texts, "expense") + assert_includes prompt, "strictly between the Expenses heading and Total Expenses or Total Expenditures" + assert_includes prompt, "row after those totals" + end + + test "rejects responses that exceed the application safety limit" do + pipeline = Warehouse::FinancialStatementExtraction::DetailedPipeline.new( + pdf_path: "unused.pdf", institution_canonical_id: "ca/on/example", + institution_name: "Example", + document_canonical_id: "ca/on/example/documents/financial-statements/2025/general", + asset_sha256: "a" * 64, fiscal_year_end: Date.new(2025, 12, 31) + ) + item = { + "flow" => "revenue", "category" => "Taxes", "label" => "Property taxes", + "raw_text" => "1", "scale" => 1, "excerpt_page" => 1, + "column_year" => "2025", "confidence" => 0.99 + } + response = { "fiscal_year" => 2025, "line_items" => Array.new(101) { |index| item.merge("label" => "Row #{index}") } } + + error = assert_raises(Warehouse::FinancialStatementExtraction::DetailedPipeline::ResponseError) do + pipeline.send(:validate_response!, response, [ 1 ], "revenue") + end + assert_equal "too many line items", error.message + end + + test "validates flow concurrency and requires isolated concurrent clients" do + attributes = pipeline_attributes + + error = assert_raises(ArgumentError) do + Warehouse::FinancialStatementExtraction::DetailedPipeline.new(**attributes, flow_concurrency: 3) + end + assert_equal "flow_concurrency must be 1 or 2", error.message + + error = assert_raises(ArgumentError) do + Warehouse::FinancialStatementExtraction::DetailedPipeline.new( + **attributes, flow_concurrency: 2, llm_client: ->(**) { nil } + ) + end + assert_equal "concurrent flow extraction requires llm_client_factory", error.message + + shared_client = ->(**) { nil } + pipeline = Warehouse::FinancialStatementExtraction::DetailedPipeline.new( + **attributes, flow_concurrency: 2, llm_client_factory: -> { shared_client } + ) + error = assert_raises(Warehouse::FinancialStatementExtraction::DetailedPipeline::ResponseError) do + pipeline.run + end + assert_equal "llm_client_factory must return a distinct callable per flow", error.message + end + + test "concurrent flow calls overlap but return deterministic results and clean excerpts" do + gates = { "revenue" => Queue.new, "expense" => Queue.new } + started = Queue.new + completed = Queue.new + paths = Queue.new + events = Queue.new + client_factory = lambda do + lambda do |prompt:, pdf_path:| + flow = prompt.include?("REQUESTED FLOW: revenue") ? "revenue" : "expense" + paths << pdf_path + started << flow + gates.fetch(flow).pop + completed << flow + detail_response(flow) + end + end + pipeline = Warehouse::FinancialStatementExtraction::DetailedPipeline.new( + **pipeline_attributes, flow_concurrency: 2, llm_client_factory: client_factory, + flow_reporter: ->(event) { events << event } + ) + + runner = Thread.new { pipeline.run } + observed = Timeout.timeout(2) { 2.times.map { started.pop }.sort } + assert_equal %w[expense revenue], observed + gates.fetch("expense") << true + assert_equal "expense", Timeout.timeout(2) { completed.pop } + gates.fetch("revenue") << true + result = Timeout.timeout(2) { runner.value } + + assert_equal %w[revenue expense], result.line_items.map { _1.fetch(:flow) } + assert_equal %w[revenue expense], result.response.fetch("details").keys + materialized_paths = 2.times.map { paths.pop } + assert materialized_paths.all? { |path| !File.exist?(path) } + reported = 2.times.map { events.pop } + assert_equal %w[expense revenue], reported.map { _1.fetch(:flow) }.sort + assert reported.all? { _1.fetch(:financial_statement_detail_flow) == "success" } + end + + test "concurrent flow failure joins its sibling and removes every excerpt" do + sibling_finished = false + paths = Queue.new + events = Queue.new + client_factory = lambda do + lambda do |prompt:, pdf_path:| + flow = prompt.include?("REQUESTED FLOW: revenue") ? "revenue" : "expense" + paths << pdf_path + raise "revenue failed" if flow == "revenue" + + sleep 0.05 + sibling_finished = true + detail_response(flow) + end + end + pipeline = Warehouse::FinancialStatementExtraction::DetailedPipeline.new( + **pipeline_attributes, flow_concurrency: 2, llm_client_factory: client_factory, + flow_reporter: ->(event) { events << event } + ) + + error = assert_raises(RuntimeError) { pipeline.run } + + assert_equal "revenue failed", error.message + assert sibling_finished + materialized_paths = 2.times.map { paths.pop } + assert materialized_paths.all? { |path| !File.exist?(path) } + reported = 2.times.map { events.pop } + assert_equal [ "failure", "success" ], reported.map { _1.fetch(:financial_statement_detail_flow) }.sort + end + + private + + def pipeline_attributes + facts = [ + fact("total_financial_assets", "100,000", 100_000_000, "financial_position"), + fact("total_liabilities", "60,000", 60_000_000, "financial_position"), + fact("net_financial_assets", "40,000", 40_000_000, "financial_position"), + fact("total_non_financial_assets", "160,000", 160_000_000, "financial_position"), + fact("accumulated_surplus", "200,000", 200_000_000, "financial_position"), + fact("total_revenue", "80,000", 80_000_000, "operations"), + fact("total_expenses", "70,000", 70_000_000, "operations"), + fact("annual_surplus", "10,000", 10_000_000, "operations") + ] + locator_result = Warehouse::FinancialStatementExtraction::PageLocator::Result.new( + page_count: 1, + page_texts: { 1 => "Revenue Property taxes 80,000 80,000 Expenses Operations 70,000 70,000" }, + position_page: 1, operations_page: 1, candidate_pages: [ 1 ], ocr_pages: [] + ) + headline = Warehouse::FinancialStatementExtraction::Pipeline::Result.new( + status: "extracted", facts:, + checks: [ { id: "source_identity", status: "pass", detail: "ok" } ], + prompt: "headline", response: { + "remeasurement_present" => false, "operations_adjustment_present" => false, + "rollforward_adjustment_present" => false + }, locator_result:, language: "en", statement_basis: "consolidated" + ) + headline_pipeline = Struct.new(:result) { def run = result }.new(headline) + page_locator = Class.new do + def with_excerpt(_pages) + Dir.mktmpdir("detailed-pipeline-test") do |directory| + path = Pathname(directory).join("excerpt.pdf") + path.write("%PDF-test") + yield path + end + end + end.new + { + pdf_path: "unused.pdf", institution_canonical_id: "ca/on/example", + institution_name: "Example", + document_canonical_id: "ca/on/example/documents/financial-statements/2025/general", + asset_sha256: "a" * 64, fiscal_year_end: Date.new(2025, 12, 31), + headline_pipeline:, page_locator: + } + end + + def detail_response(flow) + amount = flow == "revenue" ? "80,000" : "70,000" + { + "fiscal_year" => 2025, + "line_items" => [ { + "flow" => flow, "category" => flow == "revenue" ? "Taxes" : "Services", + "label" => flow == "revenue" ? "Property taxes" : "Operations", + "raw_text" => amount, "scale" => 1_000, "excerpt_page" => 1, + "column_year" => "2025", "confidence" => 0.99 + } ] + } + end + + def fact(concept, raw_text, value, statement) + { + concept:, raw_label: concept.humanize, raw_text:, value: BigDecimal(value.to_s), + scale: 1_000, statement:, source_page: 1, column_year: "2025", + extraction_confidence: BigDecimal("0.99") + } + end +end diff --git a/test/models/warehouse/financial_statement_extraction/number_parser_test.rb b/test/models/warehouse/financial_statement_extraction/number_parser_test.rb index ebe2b3eb..4ff8424f 100644 --- a/test/models/warehouse/financial_statement_extraction/number_parser_test.rb +++ b/test/models/warehouse/financial_statement_extraction/number_parser_test.rb @@ -7,18 +7,81 @@ class Warehouse::FinancialStatementExtraction::NumberParserTest < ActiveSupport: assert_equal BigDecimal("1234567"), Parser.parse("1,234,567") assert_equal BigDecimal("1234567"), Parser.parse("1 234 567") assert_equal BigDecimal("1234567.89"), Parser.parse("1\u202F234\u202F567,89") + assert_equal BigDecimal("3281940"), Parser.parse("3.281,940") + assert_equal BigDecimal("1661695"), Parser.parse("1,661.695") + assert_equal BigDecimal("3281.94"), Parser.parse("3.281,94") end test "parses parentheses and explicit minus as negative" do assert_equal BigDecimal("-1234"), Parser.parse("(1,234)") + assert_equal BigDecimal("-1234"), Parser.parse("$ (1,234)") assert_equal BigDecimal("-1234"), Parser.parse("- 1 234") end + test "repairs an opening OCR brace only in an otherwise parenthetical numeric token" do + assert_equal BigDecimal("-3900710"), Parser.parse("{3,900,710)") + + [ "{3,900,710", "3,900{710)", "{abc)", "{3,900,710}" ].each do |token| + assert_raises(Parser::ParseError) { Parser.parse(token) } + end + end + + test "repairs one OCR quote inserted between valid thousands separators" do + assert_equal BigDecimal("63121"), Parser.parse("63,',121") + assert_equal BigDecimal("-63121"), Parser.parse("$ (63,’ ,121)") + + [ "6'3121", "63,'12", "63,',12", "value 63,',121", "1,23,',456" ].each do |token| + assert_raises(Parser::ParseError) { Parser.parse(token) } + end + end + + test "repairs one OCR quote after a comma in a valid grouped integer" do + assert_equal BigDecimal("20159"), Parser.parse("20,'159") + assert_equal BigDecimal("-20159"), Parser.parse("$ (20,’159)") + + [ "20,'15", "20'159", "20,''159", "20,'1,159", "value 20,'159", "1,23,'456" ].each do |token| + assert_raises(Parser::ParseError) { Parser.parse(token) } + end + end + + test "repairs a spaced OCR quote before a valid grouped integer" do + assert_equal BigDecimal("344092"), Parser.parse("' 344,092") + assert_equal BigDecimal("344092"), Parser.parse("’\u00A0344,092") + + [ "'344,092", "3'44,092", "' 344,09", "amount ' 344,092", "'", "’" ].each do |token| + assert_raises(Parser::ParseError) { Parser.parse(token) } + end + end + + test "repairs one OCR period after a valid grouped integer" do + assert_equal BigDecimal("11140886"), Parser.parse("11,140,886.") + assert_equal BigDecimal("1234.50"), Parser.parse("1,234.50") + + [ "123.", "1,234..", "value 1,234.", "1,23,456." ].each do |token| + assert_raises(Parser::ParseError) { Parser.parse(token) } + end + end + + test "repairs visually verified lowercase ell glyphs in a grouped integer" do + assert_equal BigDecimal("11744"), Parser.parse("ll,744") + assert_equal BigDecimal("11744000"), Parser.parse("ll,744,000") + + [ "l1,744", "I1,744", "ll744", "ll,74", "value ll,744" ].each do |token| + assert_raises(Parser::ParseError) { Parser.parse(token) } + end + end + test "normalizes a positive printed net debt to negative net financial assets" do assert_equal BigDecimal("-1234"), Parser.parse("1,234", raw_label: "Net debt", concept: "net_financial_assets") assert_equal BigDecimal("-1234"), Parser.parse("1 234", raw_label: "Dette nette", concept: "net_financial_assets") end + test "keeps a combined net financial assets and net debt label positive when printed positive" do + assert_equal BigDecimal("1234"), Parser.parse( + "1 234", raw_label: "ACTIFS FINANCIERS NETS (DETTE NETTE)", concept: "net_financial_assets" + ) + end + test "normalizes positive printed deficits to negative surplus concepts" do assert_equal BigDecimal("-1250"), Parser.parse( "1 250", raw_label: "Déficit de l'exercice", concept: "annual_surplus" @@ -28,8 +91,57 @@ class Warehouse::FinancialStatementExtraction::NumberParserTest < ActiveSupport: ) end + test "does not negate a positive value when the label presents surplus and deficit alternatives" do + assert_equal BigDecimal("1998"), Parser.parse( + "1,998", raw_label: "ANNUAL (DEFICIT)/SURPLUS", concept: "annual_surplus" + ) + end + + test "does not negate unaccented French excedent and deficit alternatives" do + assert_equal BigDecimal("21684393"), Parser.parse( + "21684393", raw_label: "EXCEDENT (DEFICIT) ACCUMULE", concept: "accumulated_surplus" + ) + assert_equal BigDecimal("31445383"), Parser.parse( + "31 445 383", raw_label: "EXCÉDENT (DÉFICIT) ACCUMULÉ", concept: "accumulated_surplus" + ) + end + test "does not conflate dash with zero" do - assert_raises(Parser::ParseError) { Parser.parse("—") } + [ "-", "‐", "‒", "–", "—", "−", "--", "---", "‒‒", "−−", "$ -", "$ ‒", "$ −", "$ --", "$ ---" ].each do |dash| + assert Parser.null_marker?(dash), "expected #{dash.inspect} to be a null marker" + assert_raises(Parser::ParseError) { Parser.parse(dash) } + end + [ "----", "1---2", "1‒2", "1−2", "−1,234", "value ‒", "value −" ].each do |token| + refute Parser.null_marker?(token) + assert_raises(Parser::ParseError) { Parser.parse(token) } + end + assert Parser.null_marker?("=") + refute Parser.null_marker?("0") assert_equal BigDecimal("0"), Parser.parse("0") end + + test "does not treat embedded Unicode hyphens as null markers or numeric signs" do + [ "1‐2", "‐123" ].each do |token| + refute Parser.null_marker?(token) + assert_raises(Parser::ParseError) { Parser.parse(token) } + end + end + + test "treats isolated OCR quote glyphs as null markers rather than numeric zero" do + [ '"', "“", "”", "„", "‟", "″" ].each do |glyph| + assert Parser.null_marker?(glyph), "expected #{glyph.inspect} to be a null marker" + assert_raises(Parser::ParseError) { Parser.parse(glyph) } + end + end + + test "treats an isolated OCR period as a null marker without broad punctuation repair" do + assert Parser.null_marker?(".") + assert Parser.null_marker?(" . ") + assert_raises(Parser::ParseError) { Parser.parse(".") } + + [ "..", "...", ".0", "0.", "value ." ].each do |token| + refute Parser.null_marker?(token) + assert_raises(Parser::ParseError) { Parser.parse(token) } + end + end end diff --git a/test/models/warehouse/financial_statement_extraction/ocr_text_cache_test.rb b/test/models/warehouse/financial_statement_extraction/ocr_text_cache_test.rb new file mode 100644 index 00000000..d477ef10 --- /dev/null +++ b/test/models/warehouse/financial_statement_extraction/ocr_text_cache_test.rb @@ -0,0 +1,123 @@ +require "test_helper" + +class Warehouse::FinancialStatementExtraction::OcrTextCacheTest < ActiveSupport::TestCase + setup { Warehouse::FinancialStatementExtraction::OcrTextCache.reset_statistics! } + + test "reuses exact content-addressed OCR text and reports cumulative hits" do + Dir.mktmpdir do |directory| + source = Pathname(directory).join("source.pdf") + source.write("immutable pdf bytes") + root = Pathname(directory).join("cache") + events = [] + reporter = ->(event) { events << event } + calls = 0 + options = { dpi: 300, preprocessing: { psm: 6 } } + + first = build_cache(root:, source:, reporter:).fetch(page: 7, mode: "table", options:) do + calls += 1 + "table text\n" + end + second = build_cache(root:, source:, reporter:).fetch(page: 7, mode: "table", options:) do + flunk "a cache hit must not run OCR" + end + + assert_equal "table text\n", first + assert_equal first, second + assert_equal 1, calls + assert_equal %w[miss hit], events.pluck(:ocr_cache) + assert_equal({ "miss" => 1 }, events.first.fetch(:counts)) + assert_equal({ "miss" => 1, "hit" => 1 }, events.second.fetch(:counts)) + end + end + + test "separates source content page mode and OCR options" do + Dir.mktmpdir do |directory| + root = Pathname(directory).join("cache") + source_a = Pathname(directory).join("a.pdf").tap { _1.write("asset a") } + source_b = Pathname(directory).join("b.pdf").tap { _1.write("asset b") } + calls = 0 + fetch = lambda do |source:, page:, mode:, dpi:| + build_cache(root:, source:).fetch(page:, mode:, options: { dpi: }) do + calls += 1 + "result #{calls}" + end + end + + assert_equal "result 1", fetch.call(source: source_a, page: 1, mode: "plain", dpi: 300) + assert_equal "result 2", fetch.call(source: source_b, page: 1, mode: "plain", dpi: 300) + assert_equal "result 3", fetch.call(source: source_a, page: 2, mode: "plain", dpi: 300) + assert_equal "result 4", fetch.call(source: source_a, page: 1, mode: "table", dpi: 300) + assert_equal "result 5", fetch.call(source: source_a, page: 1, mode: "plain", dpi: 400) + assert_equal 5, calls + end + end + + test "ignores a corrupted entry and atomically replaces it" do + Dir.mktmpdir do |directory| + source = Pathname(directory).join("source.pdf").tap { _1.write("asset") } + root = Pathname(directory).join("cache") + cache = build_cache(root:, source:) + assert_equal "first", cache.fetch(page: 1, mode: "plain", options: {}) { "first" } + entry = root.glob("*/*.json").sole + entry.write("not json") + + calls = 0 + result = build_cache(root:, source:).fetch(page: 1, mode: "plain", options: {}) do + calls += 1 + "recomputed" + end + + assert_equal "recomputed", result + assert_equal 1, calls + assert_equal "recomputed", JSON.parse(entry.read).fetch("text") + end + end + + test "does not cache failed OCR" do + Dir.mktmpdir do |directory| + source = Pathname(directory).join("source.pdf").tap { _1.write("asset") } + root = Pathname(directory).join("cache") + cache = build_cache(root:, source:) + + assert_raises(Errno::ENOENT) do + cache.fetch(page: 1, mode: "plain", options: {}) { raise Errno::ENOENT, "tesseract" } + end + assert_empty root.glob("*/*.json") + end + end + + test "concurrent callers compute once and leave one valid entry" do + Dir.mktmpdir do |directory| + source = Pathname(directory).join("source.pdf").tap { _1.write("asset") } + root = Pathname(directory).join("cache") + calls = 0 + mutex = Mutex.new + start = Queue.new + results = Queue.new + workers = 2.times.map do + Thread.new do + start.pop + result = build_cache(root:, source:).fetch(page: 9, mode: "table", options: { dpi: 300 }) do + mutex.synchronize { calls += 1 } + sleep 0.05 + "shared result" + end + results << result + end + end + 2.times { start << true } + workers.each(&:join) + + assert_equal [ "shared result", "shared result" ], 2.times.map { results.pop }.sort + assert_equal 1, calls + assert_equal 1, root.glob("*/*.json").length + assert_equal "shared result", JSON.parse(root.glob("*/*.json").sole.read).fetch("text") + end + end + + private + + def build_cache(root:, source:, reporter: ->(_event) { }) + Warehouse::FinancialStatementExtraction::OcrTextCache.new(root:, source_path: source, reporter:) + end +end diff --git a/test/models/warehouse/financial_statement_extraction/page_locator_test.rb b/test/models/warehouse/financial_statement_extraction/page_locator_test.rb index 723859e7..2f4242c6 100644 --- a/test/models/warehouse/financial_statement_extraction/page_locator_test.rb +++ b/test/models/warehouse/financial_statement_extraction/page_locator_test.rb @@ -1,6 +1,44 @@ require "test_helper" class Warehouse::FinancialStatementExtraction::PageLocatorTest < ActiveSupport::TestCase + test "shares exact table OCR through the persistent content cache" do + Dir.mktmpdir do |directory| + source = Pathname(directory).join("source.pdf").tap { _1.write("immutable pdf bytes") } + cache_root = Pathname(directory).join("cache") + calls = [] + first = Warehouse::FinancialStatementExtraction::PageLocator.new( + source, ocr_cache_root: cache_root, ocr_cache_reporter: ->(_event) { } + ) + first.define_singleton_method(:perform_table_ocr) do |page| + calls << page + "cached table OCR #{page}" + end + second = Warehouse::FinancialStatementExtraction::PageLocator.new( + source, ocr_cache_root: cache_root, ocr_cache_reporter: ->(_event) { } + ) + second.define_singleton_method(:perform_table_ocr) do |_page| + flunk "persistent cache hit must not invoke OCR tools" + end + + assert_equal "cached table OCR 7", first.ocr_table_page(7) + assert_equal "cached table OCR 7", second.ocr_table_page(7) + assert_equal [ 7 ], calls + end + end + + test "memoizes table OCR by physical page" do + locator = Warehouse::FinancialStatementExtraction::PageLocator.new("unused.pdf") + calls = [] + locator.define_singleton_method(:perform_table_ocr) do |page| + calls << page + "table OCR #{page}" + end + + assert_equal "table OCR 7", locator.ocr_table_page(7) + assert_equal "table OCR 7", locator.ocr_table_page("7") + assert_equal [ 7 ], calls + end + test "selects the first primary statements after the auditor report" do locator = Warehouse::FinancialStatementExtraction::PageLocator.new("unused.pdf") pages = { @@ -15,4 +53,176 @@ class Warehouse::FinancialStatementExtraction::PageLocatorTest < ActiveSupport:: assert_equal 5, locator.send(:locate_page, pages, :position) assert_equal 6, locator.send(:locate_page, pages, :operations) end + + test "does not select wrapped auditor prose after an index mentions the auditor report" do + locator = Warehouse::FinancialStatementExtraction::PageLocator.new("unused.pdf") + pages = { + 2 => "Contents\nIndependent Auditor's Report 1\nConsolidated Statement of Financial Position 3", + 4 => <<~TEXT, + Independent Auditor's Report + Opinion + We audited the consolidated financial statements, which comprise the + consolidated statement of financial position as at December 31, 2022. + TEXT + 6 => <<~TEXT + Municipality of Northern Village of Air Ronge + Consolidated Statement of Financial Position + As at December 31, 2022 + 2022 2021 + Financial Assets 7,109,579 6,412,443 + TEXT + } + + assert_equal 6, locator.send(:locate_page, pages, :position) + end + + test "does not select an index page ahead of scanned primary statements" do + locator = Warehouse::FinancialStatementExtraction::PageLocator.new("unused.pdf") + pages = { + 2 => <<~TEXT, + City + Index to the Consolidated Financial Statements + Consolidated Statement of Financial Position + Consolidated Statement of Financial Activities + 4 5 6 7 8 9 + TEXT + 4 => "City\nConsolidated Statement of Financial Position\n2025 2024\nAssets 100 90", + 5 => "City\nConsolidated Statement of Financial Activities\n2025 2024\nRevenue 80 75" + } + + assert_equal 4, locator.send(:locate_page, pages, :position) + assert_equal 5, locator.send(:locate_page, pages, :operations) + end + + test "requests OCR instead of accepting an exhibit by-fund position statement" do + locator = Warehouse::FinancialStatementExtraction::PageLocator.new("unused.pdf") + pages = { + 6 => "", + 7 => "City\nConsolidated Statement of Operations\n2025 2024\nRevenue 80 75", + 31 => <<~TEXT + CITY OF EXAMPLE Exhibit 1 + Statement of Financial Position - By Fund + 2025 2024 + Financial assets 100 90 + TEXT + } + + assert locator.send(:needs_ocr?, pages) + + pages[6] = <<~TEXT + CITY OF EXAMPLE + Consolidated Statement of Financial Position + 2025 2024 + FINANCIAL ASSETS + Cash and cash equivalents (Note 2) 100 90 + TEXT + assert_equal 6, locator.send(:locate_page, pages, :position) + end + + test "does not treat French auditor prose as a primary statement heading" do + locator = Warehouse::FinancialStatementExtraction::PageLocator.new("unused.pdf") + pages = { + 4 => "Village\nRAPPORT\nNous avons audité l'état de la situation financière et l'état des résultats.", + 7 => "Village\nÉTAT DES RÉSULTATS\nEXERCICE TERMINÉ LE 31 DÉCEMBRE 2023\n2023 2023 2022\nRevenus 1 10 9", + 8 => "Village\nÉTAT DE LA SITUATION FINANCIÈRE\nAU 31 DÉCEMBRE 2023\n2023 2022\nActifs 10 9" + } + + assert_equal 7, locator.send(:locate_page, pages, :operations) + assert_equal 8, locator.send(:locate_page, pages, :position) + end + + test "requests OCR when only auditor prose identifies a blank primary statement" do + locator = Warehouse::FinancialStatementExtraction::PageLocator.new("unused.pdf") + pages = { + 3 => "Independent Auditors' Report\nWe audited the consolidated statement of financial position.", + 5 => "", + 6 => "City\nConsolidated Statement of Operations\n2025 2024\nRevenue 80 75" + } + + assert locator.send(:needs_ocr?, pages) + end + + test "tolerates one short OCR noise token in a primary financial position title" do + locator = Warehouse::FinancialStatementExtraction::PageLocator.new("unused.pdf") + pages = { + 2 => "Independent Auditors' Report\nWe audited the statement of financial PI position.", + 4 => "Town\nSTATEMENT OF FINANCIAL PI POSITION\n2011 2010\nFinancial assets 100 90", + 8 => "Notes to Financial Statements\nThe statement of financial PI position includes several estimates." + } + + assert_equal 4, locator.send(:locate_page, pages, :position) + end + + test "does not tolerate an unbounded OCR phrase in a financial position title" do + locator = Warehouse::FinancialStatementExtraction::PageLocator.new("unused.pdf") + pages = { + 4 => "Town\nSTATEMENT OF FINANCIAL VERY NOISY POSITION\n2011 2010\nFinancial assets 100 90" + } + + assert_nil locator.send(:locate_page, pages, :position) + end + + test "recognizes malformed dense-table OCR numbers that need a layout retry" do + locator = Warehouse::FinancialStatementExtraction::PageLocator.new("unused.pdf") + + assert_equal 1, locator.send(:malformed_ocr_number_count, "Capital transfers 29,522.21]") + assert_equal 0, locator.send(:malformed_ocr_number_count, "Capital transfers 29,522,211") + end + + test "repairs single-space OCR digit fragments without merging spaced columns" do + locator = Warehouse::FinancialStatementExtraction::PageLocator.new("unused.pdf") + text = "Net assets 494 468 556,508\nCapital 67.71 7 165,727\n" + + assert_equal "Net assets 494468 556,508\nCapital 67.717 165,727\n", + locator.send(:normalize_table_ocr_digit_fragments, text) + end + + test "repairs only bounded leading table OCR digit confusables" do + locator = Warehouse::FinancialStatementExtraction::PageLocator.new("unused.pdf") + text = "Total L,747,384 I,234,567 l,111,222 1,747,384 AL,747,384 L,747\n" + + assert_equal "Total 1,747,384 1,234,567 1,111,222 1,747,384 AL,747,384 L,747\n", + locator.send(:normalize_table_ocr_digit_fragments, text) + end + + test "computes the inverse correction for rotated scanned pages" do + locator = Warehouse::FinancialStatementExtraction::PageLocator.new("unused.pdf") + locator.instance_variable_set(:@page_rotation, 270) + success = Object.new + success.define_singleton_method(:success?) { true } + + capture = lambda do |*arguments| + assert_equal [ "magick", "/tmp/source.png", "-rotate", "90", "/tmp/page-oriented.png" ], arguments + [ "", "", success ] + end + Open3.stub(:capture3, capture) do + result = locator.send( + :correct_ocr_orientation, "/tmp/source.png", directory: "/tmp", name: "page" + ) + + assert_equal "/tmp/page-oriented.png", result + end + end + + test "does not rotate a landscape source that renders upright" do + locator = Warehouse::FinancialStatementExtraction::PageLocator.new("unused.pdf") + locator.instance_variable_set(:@page_rotation, 90) + locator.instance_variable_set(:@source_page_landscape, true) + + assert_equal "/tmp/source.png", locator.send( + :correct_ocr_orientation, "/tmp/source.png", directory: "/tmp", name: "page" + ) + end + + test "does not rotate a 180 degree source that pdftoppm renders upright" do + locator = Warehouse::FinancialStatementExtraction::PageLocator.new("unused.pdf") + locator.instance_variable_set(:@page_rotation, 180) + locator.instance_variable_set(:@source_page_landscape, false) + + Open3.stub(:capture3, ->(*) { flunk "ImageMagick should not be called" }) do + assert_equal "/tmp/source.png", locator.send( + :correct_ocr_orientation, "/tmp/source.png", directory: "/tmp", name: "page" + ) + end + end end diff --git a/test/models/warehouse/financial_statement_extraction/pipeline_test.rb b/test/models/warehouse/financial_statement_extraction/pipeline_test.rb index 4a1a5e84..cdb5d709 100644 --- a/test/models/warehouse/financial_statement_extraction/pipeline_test.rb +++ b/test/models/warehouse/financial_statement_extraction/pipeline_test.rb @@ -1,6 +1,76 @@ require "test_helper" class Warehouse::FinancialStatementExtraction::PipelineTest < ActiveSupport::TestCase + test "limits headline extraction to primary statements and a relevant supporting schedule" do + locator = Warehouse::FinancialStatementExtraction::PageLocator::Result.new( + page_count: 19, + page_texts: { + 6 => "Statement of Operations 2010 Revenues Expenses", + 7 => "Statement of Financial Position 2010", + 8 => "Statement of Cash Flows 2010 revenues expenses", + 19 => "Schedule 3 Revenue and Expenditures 2010\nRevenues 3,000 Expenses 2,000" + }, + position_page: 7, operations_page: 6, candidate_pages: [ 5, 6, 7, 8 ], ocr_pages: [] + ) + pipeline = Warehouse::FinancialStatementExtraction::Pipeline.allocate + pipeline.instance_variable_set(:@fiscal_year_end, Date.new(2010, 12, 31)) + + limited = pipeline.send(:primary_statement_locator, locator) + + assert_equal [ 6, 7, 19 ], limited.candidate_pages + end + + test "does not manufacture a year that was absent from the model's cited heading" do + normalized = Warehouse::FinancialStatementExtraction::Pipeline.normalize_column_year( + "Actual", fiscal_year: 2024, page_text: "2024 2024 2023\nBudget Actual Actual" + ) + + assert_equal "Actual", normalized + end + + test "prompt defines source-backed single-component totals and explicit provenance" do + pipeline = Warehouse::FinancialStatementExtraction::Pipeline.allocate + pipeline.instance_variable_set(:@institution_name, "Example") + pipeline.instance_variable_set(:@institution_canonical_id, "ca/nl/example") + pipeline.instance_variable_set(:@document_canonical_id, "ca/nl/example/documents/financial-statements/2025/general") + pipeline.instance_variable_set(:@fiscal_year_end, Date.new(2025, 12, 31)) + located = Warehouse::FinancialStatementExtraction::PageLocator::Result.new( + page_count: 2, page_texts: { 1 => "Position", 2 => "Operations" }, + position_page: 1, operations_page: 2, candidate_pages: [ 1, 2 ], ocr_pages: [] + ) + + prompt = pipeline.send(:build_prompt, located) + + assert_includes prompt, "exactly one printed component" + assert_includes prompt, "set the matching *_single_component boolean true" + assert_includes prompt, "confidence no higher than 0.90" + end + + test "requires a flagged single-component fact and caps its confidence" do + pipeline = Warehouse::FinancialStatementExtraction::Pipeline.allocate + pipeline.instance_variable_set(:@fiscal_year_end, Date.new(2025, 12, 31)) + located = Warehouse::FinancialStatementExtraction::PageLocator::Result.new( + page_count: 1, page_texts: { 1 => "Position" }, position_page: 1, + operations_page: 1, candidate_pages: [ 1 ], ocr_pages: [] + ) + response = { + "language" => "en", "statement_basis" => "consolidated", "fiscal_year" => 2025, + "total_financial_assets_single_component" => true, + "facts" => [ fact("total_liabilities", "Liabilities", "1", 1) ] + } + + error = assert_raises(Warehouse::FinancialStatementExtraction::Pipeline::ResponseError) do + pipeline.send(:validate_response!, response, located) + end + assert_includes error.message, "lacks total_financial_assets" + + response["facts"] = [ fact("total_financial_assets", "Accounts receivable", "1", 1) ] + error = assert_raises(Warehouse::FinancialStatementExtraction::Pipeline::ResponseError) do + pipeline.send(:validate_response!, response, located) + end + assert_includes error.message, "confidence exceeds 0.90" + end + Locator = Struct.new(:result) do def locate = result def with_excerpt(*) = yield(Pathname("excerpt.pdf")) @@ -48,6 +118,17 @@ def with_excerpt(*) = yield(Pathname("excerpt.pdf")) assert_equal BigDecimal("-9602000000"), result.facts.index_by { |item| item[:concept] } .fetch("net_financial_assets").fetch(:value) assert_equal "pass", result.checks.find { |check| check[:id] == "source_identity" }.fetch(:status) + + stale_response = response.deep_dup + stale_response.fetch("facts").each { _1["excerpt_page"] += 5 } + source_pages = result.facts.to_h { [ _1.fetch(:concept), _1.fetch(:source_page) ] } + revalidated = Warehouse::FinancialStatementExtraction::Pipeline.new( + pdf_path: pdf, institution_canonical_id: "ca/on/example", institution_name: "Example", + document_canonical_id: "ca/on/example/documents/financial-statements/2025/general", + asset_sha256: sha256, fiscal_year_end: Date.new(2025, 12, 31), + page_locator: Locator.new(located), llm_client: ->(**) { raise "model must not be called" } + ).revalidate(response: stale_response, source_pages:) + assert_equal "extracted", revalidated.status end end diff --git a/test/models/warehouse/financial_statement_extraction/prairie_failed_candidate_filter_test.rb b/test/models/warehouse/financial_statement_extraction/prairie_failed_candidate_filter_test.rb new file mode 100644 index 00000000..9e26ddd7 --- /dev/null +++ b/test/models/warehouse/financial_statement_extraction/prairie_failed_candidate_filter_test.rb @@ -0,0 +1,246 @@ +require "test_helper" + +class Warehouse::FinancialStatementExtraction::PrairieFailedCandidateFilterTest < ActiveSupport::TestCase + setup do + @release = Warehouse::InstitutionRelease.create!( + version: "2026-08-30", effective_on: Date.new(2026, 8, 30), schema_version: "1.0", + published_at: Time.utc(2026, 8, 30), geography_vintage: 2021, attribution: "Test" + ) + @source = Warehouse::InstitutionSource.create!( + institution_release: @release, canonical_id: "ca/sources/prairie-filter-test", + publisher_name: "Test", title_en: "Test", url: "https://example.test/source", + retrieved_at: @release.published_at, languages: [ "en" ] + ) + end + + test "includes only failed municipality-years without an approved variant" do + excluded_failed = create_document("ca/ab/approved-elsewhere", 2025, "a" * 64, "general") + approved_variant = create_document("ca/ab/approved-elsewhere", 2025, "b" * 64, "audited") + failed_only = create_document("ca/ab/failed-only", 2024, "c" * 64, "general") + rejected_failed = create_document("ca/ab/rejected-only", 2023, "d" * 64, "general") + rejected_variant = create_document("ca/ab/rejected-only", 2023, "e" * 64, "audited") + + create_extraction(excluded_failed, "a" * 64, "failed") + create_extraction(approved_variant, "b" * 64, "approved") + create_extraction(failed_only, "c" * 64, "failed") + create_extraction(rejected_failed, "d" * 64, "failed") + create_extraction(rejected_variant, "e" * 64, "rejected") + + candidates = Warehouse::FinancialStatementExtraction::CandidateSet.new( + release: @release, provinces: [ "ab" ] + ).each.to_a + by_document = candidates.index_by(&:document_canonical_id) + filter = Warehouse::FinancialStatementExtraction::FailedCandidateFilter.new( + release: @release, province: "ab", candidates: + ) + + refute filter.eligible?(by_document.fetch(excluded_failed.canonical_id)) + refute filter.eligible?(by_document.fetch(approved_variant.canonical_id)) + assert filter.eligible?(by_document.fetch(failed_only.canonical_id)) + assert filter.eligible?(by_document.fetch(rejected_failed.canonical_id)) + refute filter.eligible?(by_document.fetch(rejected_variant.canonical_id)) + + report = filter.report + assert_equal 3, report.fetch(:aggregated_failure_count) + assert_equal 2, report.fetch(:included_failure_count) + assert_equal 2, report.fetch(:eligible_document_count) + assert_equal 2, report.fetch(:public_slot_count) + assert_equal 1, report.fetch(:approved_elsewhere_excluded_count) + assert_equal 0, report.fetch(:review_pending_elsewhere_excluded_count) + assert_equal 0, report.fetch(:duplicate_slot_excluded_count) + assert_equal 0, report.fetch(:unmatched_failure_count) + assert report.fetch(:reconciled) + assert_equal [ "ca/ab/approved-elsewhere" ], + report.dig(:approved_elsewhere_excluded, 0, :candidate_institutions) + end + + test "uses public years for approved coverage and selects one ranked variant" do + covered_failure = create_document( + "ca/ab/year-covered", 2025, "1" * 64, "general", + fiscal_period_end: Date.new(2025, 12, 31) + ) + approved_different_date = create_document( + "ca/ab/year-covered", 2025, "2" * 64, "consolidated", + fiscal_period_end: Date.new(2025, 3, 31) + ) + consolidated = create_document("ca/ab/ranked", 2024, "3" * 64, "consolidated") + general = create_document("ca/ab/ranked", 2024, "4" * 64, "general") + create_extraction(covered_failure, "1" * 64, "failed") + create_extraction(approved_different_date, "2" * 64, "approved") + create_extraction(consolidated, "3" * 64, "failed") + create_extraction(general, "4" * 64, "failed") + + candidates = Warehouse::FinancialStatementExtraction::CandidateSet.new( + release: @release, provinces: [ "ab" ] + ).each.to_a + by_document = candidates.index_by(&:document_canonical_id) + filter = Warehouse::FinancialStatementExtraction::FailedCandidateFilter.new( + release: @release, province: "ab", candidates: + ) + + refute filter.eligible?(by_document.fetch(covered_failure.canonical_id)) + assert filter.eligible?(by_document.fetch(consolidated.canonical_id)) + refute filter.eligible?(by_document.fetch(general.canonical_id)) + report = filter.report + assert_equal 3, report.fetch(:aggregated_failure_count) + assert_equal 1, report.fetch(:included_failure_count) + assert_equal 1, report.fetch(:approved_elsewhere_excluded_count) + assert_equal 0, report.fetch(:review_pending_elsewhere_excluded_count) + assert_equal 1, report.fetch(:duplicate_slot_excluded_count) + assert_equal 1, report.fetch(:eligible_document_count) + assert_equal 1, report.fetch(:public_slot_count) + assert report.fetch(:reconciled) + superseded = report.fetch(:duplicate_slot_excluded).sole.fetch(:superseded_slots).sole + assert_equal consolidated.canonical_id, superseded.fetch(:winner_document_canonical_id) + assert_equal "failed", superseded.fetch(:winner_status) + end + + test "counts a shared failed identity once while exposing each public slot winner" do + first = create_document("ca/ab/shared-first", 2024, "5" * 64, "consolidated") + second = create_document("ca/ab/shared-second", 2024, "5" * 64, "consolidated") + create_extraction(first, "5" * 64, "failed") + candidates = Warehouse::FinancialStatementExtraction::CandidateSet.new( + release: @release, provinces: [ "ab" ] + ).each.to_a + filter = Warehouse::FinancialStatementExtraction::FailedCandidateFilter.new( + release: @release, province: "ab", candidates: + ) + + assert candidates.all? { filter.eligible?(_1) } + assert_equal 1, filter.report.fetch(:included_failure_count) + assert_equal 2, filter.report.fetch(:eligible_document_count) + assert_equal 2, filter.report.fetch(:public_slot_count) + assert filter.report.fetch(:reconciled) + end + + test "excludes a failed variant while its public year is awaiting review" do + failed = create_document("ca/ab/review-pending", 2024, "7" * 64, "general") + awaiting_review = create_document( + "ca/ab/review-pending", 2024, "8" * 64, "consolidated", + fiscal_period_end: Date.new(2024, 3, 31) + ) + create_extraction(failed, "7" * 64, "failed") + create_extraction(awaiting_review, "8" * 64, "needs_review") + candidates = Warehouse::FinancialStatementExtraction::CandidateSet.new( + release: @release, provinces: [ "ab" ] + ).each.to_a + filter = Warehouse::FinancialStatementExtraction::FailedCandidateFilter.new( + release: @release, province: "ab", candidates: + ) + + refute filter.eligible?(candidates.find { _1.document_canonical_id == failed.canonical_id }) + assert_equal 0, filter.report.fetch(:included_failure_count) + assert_equal 1, filter.report.fetch(:review_pending_elsewhere_excluded_count) + assert filter.report.fetch(:reconciled) + end + + test "reports persisted failures absent from the candidate set" do + document = create_document("ca/ab/unmatched", 2024, "6" * 64, "consolidated") + create_extraction(document, "6" * 64, "failed") + filter = Warehouse::FinancialStatementExtraction::FailedCandidateFilter.new( + release: @release, province: "ab", candidates: [] + ) + + assert_equal 1, filter.unmatched_keys.length + assert_equal 1, filter.report.fetch(:unmatched_failure_count) + assert filter.report.fetch(:reconciled) + end + + test "targets failed parser versions while keeping slot coverage parser agnostic" do + target = create_document("ca/ab/parser-target", 2024, "9" * 64, "general") + pending_sibling = create_document("ca/ab/parser-target", 2024, "a" * 64, "consolidated") + other_failure = create_document("ca/ab/other-parser", 2023, "b" * 64, "consolidated") + create_extraction(target, "9" * 64, "failed", parser: "target-v1") + create_extraction(pending_sibling, "a" * 64, "needs_review", parser: "other-v1") + create_extraction(other_failure, "b" * 64, "failed", parser: "other-v1") + candidates = Warehouse::FinancialStatementExtraction::CandidateSet.new( + release: @release, provinces: [ "ab" ] + ).each.to_a + filter = Warehouse::FinancialStatementExtraction::FailedCandidateFilter.new( + release: @release, province: "ab", candidates:, parser_versions: [ "target-v1" ] + ) + + assert_equal [ "target-v1" ], filter.report.fetch(:failed_parser_versions) + assert_equal 1, filter.report.fetch(:aggregated_failure_count) + assert_equal 0, filter.report.fetch(:included_failure_count) + assert_equal 1, filter.report.fetch(:review_pending_elsewhere_excluded_count) + assert filter.report.fetch(:reconciled) + end + + test "targets headline failures while keeping publication coverage pinned to detailed rows" do + headline_target = create_document("ca/ab/headline-target", 2024, "c" * 64, "general") + detailed_failure = create_document("ca/ab/detailed-target", 2023, "d" * 64, "general") + covered_headline = create_document("ca/ab/headline-covered", 2022, "e" * 64, "general") + covered_detailed = create_document("ca/ab/headline-covered", 2022, "f" * 64, "consolidated") + create_extraction(headline_target, "c" * 64, "failed", extractor: :headline) + create_extraction(detailed_failure, "d" * 64, "failed") + create_extraction(covered_headline, "e" * 64, "failed", extractor: :headline) + create_extraction(covered_detailed, "f" * 64, "approved") + candidates = Warehouse::FinancialStatementExtraction::CandidateSet.new( + release: @release, provinces: [ "ab" ] + ).each.to_a + by_document = candidates.index_by(&:document_canonical_id) + + detailed_filter = Warehouse::FinancialStatementExtraction::FailedCandidateFilter.new( + release: @release, province: "ab", candidates: + ) + headline_filter = Warehouse::FinancialStatementExtraction::FailedCandidateFilter.new( + release: @release, province: "ab", candidates:, + failed_extractor_version: Warehouse::FinancialStatementExtraction::Pipeline::EXTRACTOR_VERSION + ) + + assert detailed_filter.eligible?(by_document.fetch(detailed_failure.canonical_id)) + refute detailed_filter.eligible?(by_document.fetch(headline_target.canonical_id)) + assert headline_filter.eligible?(by_document.fetch(headline_target.canonical_id)) + refute headline_filter.eligible?(by_document.fetch(detailed_failure.canonical_id)) + refute headline_filter.eligible?(by_document.fetch(covered_headline.canonical_id)) + assert_equal Warehouse::FinancialStatementExtraction::Pipeline::EXTRACTOR_VERSION, + headline_filter.report.fetch(:failed_extractor_version) + assert_equal 1, headline_filter.report.fetch(:included_failure_count) + assert_equal 1, headline_filter.report.fetch(:approved_elsewhere_excluded_count) + assert headline_filter.report.fetch(:reconciled) + end + + private + + def create_document(canonical_id, year, sha, variant, fiscal_period_end: Date.new(year, 12, 31)) + institution = @release.institutions.find_or_create_by!(canonical_id:) do |row| + row.assign_attributes( + institution_source: @source, name_en: canonical_id.split("/").last.titleize, + institution_type: "government", government_level: "municipal", status: "active" + ) + end + document = Warehouse::InstitutionDocument.create!( + institution_release: @release, institution:, institution_source: @source, + canonical_id: "#{canonical_id}/documents/financial-statements/#{year}/#{variant}", + document_type: "financial-statements", document_variant: variant, + fiscal_period_end: + ) + Warehouse::InstitutionDocumentAsset.create!( + institution_release: @release, institution_document: document, content_sha256: sha, + asset_role: "final", preferred: true, download_url: "https://example.test/#{sha}.pdf", + retrieved_at: @release.published_at, archive_path: "sha256/#{sha.first(2)}/#{sha}.pdf", + mime_type: "application/pdf", byte_size: 100, rights_status: "metadata_only" + ) + document + end + + def create_extraction(document, sha, status, parser: nil, extractor: :detailed) + extraction = Warehouse::FinancialStatementExtraction.create!( + institution_release: @release, institution_canonical_id: document.institution.canonical_id, + document_canonical_id: document.canonical_id, asset_sha256: sha, + fiscal_year_end: document.fiscal_period_end, + extractor_version: extractor == :headline ? + Warehouse::FinancialStatementExtraction::Pipeline::EXTRACTOR_VERSION : + Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION, + status: status.in?(%w[approved rejected]) ? "extracted" : status, + llm_response_snapshot: parser ? { "parser" => parser } : nil, + check_results: [ { id: "deterministic_parser", status: status == "approved" ? "pass" : "fail", + detail: "saved" } ] + ) + reviewer = Warehouse::FinancialStatementExtraction::Reviewer::REVIEWER + extraction.approve!(reviewer:) if status == "approved" + extraction.reject!(reviewer:) if status == "rejected" + extraction + end +end diff --git a/test/models/warehouse/financial_statement_extraction/quebec_form_pipeline_test.rb b/test/models/warehouse/financial_statement_extraction/quebec_form_pipeline_test.rb new file mode 100644 index 00000000..86b1a02e --- /dev/null +++ b/test/models/warehouse/financial_statement_extraction/quebec_form_pipeline_test.rb @@ -0,0 +1,111 @@ +require "test_helper" + +class Warehouse::FinancialStatementExtraction::QuebecFormPipelineTest < ActiveSupport::TestCase + test "reads current actual cells from the standardized operations form" do + page = <<~TEXT + ÉTAT DES RÉSULTATS + Budget Réalisations + 2023 2023 2022 + Revenus + Taxes 1 100 120 110 + Quotes-parts 3 (2) 1 + 13 100 118 111 + Charges + Administration générale 14 80 90 70 + 24 80 90 70 + Excédent de l'exercice 25 20 28 41 + Solde redressé 28 200 159 + Excédent accumulé 29 228 200 + TEXT + + form = Warehouse::FinancialStatementExtraction::QuebecFormPipeline::FormPage.new( + page, 2023, current_column: 1 + ) + + assert_equal "120", form.fetch(1).fetch(:raw_text) + assert_equal "(2)", form.fetch(3).fetch(:raw_text) + assert_equal "118", form.fetch(13).fetch(:raw_text) + assert_equal "Administration générale", form.fetch(14).fetch(:label) + end + + test "reads current cells when older forms omit the budget column" do + page = <<~TEXT + ÉTAT CONSOLIDÉ DES RÉSULTATS + Réalisations + 2020 2019 + Revenus + Taxes 1 120 110 + 13 120 110 + Charges + Administration générale 14 90 70 + 24 90 70 + Excédent de l'exercice 25 30 40 + Solde redressé 28 200 160 + Excédent accumulé 29 230 200 + TEXT + + form = Warehouse::FinancialStatementExtraction::QuebecFormPipeline::FormPage.new( + page, 2020, current_column: 1 + ) + + assert_equal "120", form.fetch(1).fetch(:raw_text) + assert_equal "90", form.fetch(14).fetch(:raw_text) + assert_equal "2020", form.fetch(25).fetch(:column_year) + end + + test "prefers the standardized S7 form when its OCR heading is malformed" do + located = Warehouse::FinancialStatementExtraction::PageLocator::Result.new( + page_count: 12, + page_texts: { 7 => "ÉrRr oes nÉsulrRrs\nRapport financier 2025 | S7 I", 8 => "position" }, + position_page: 11, operations_page: 10, candidate_pages: [ 10, 11 ], ocr_pages: [] + ) + pipeline = Warehouse::FinancialStatementExtraction::QuebecFormPipeline.new( + pdf_path: "unused.pdf", institution_canonical_id: "ca/qc/example", + document_canonical_id: "ca/qc/example/documents/financial-statements/2025/general", + asset_sha256: "unused", fiscal_year_end: Date.new(2025, 12, 31) + ) + + preferred = pipeline.send(:prefer_mamh_form_pages, located) + + assert_equal 7, preferred.operations_page + assert_equal 8, preferred.position_page + assert_equal [ 6, 7, 8, 9 ], preferred.candidate_pages + end + + test "reads detailed actual line items from thresholded MAMH table OCR" do + text = <<~TEXT + 2025 2025 2024 + Revenus + Taxes 5198100 5129613 4877684 + Autres revenus 10 3600 91608 84215 + 13 6322000 6688008 8235164 + Charges + Administration générale 14 1310250 1259235 1345416 + 24 5869900 6872536 6051550 + Excédent (déficit) lié aux activités 25 452100 (184528) 2183614 + TEXT + page = Warehouse::FinancialStatementExtraction::QuebecFormPipeline::OcrFormPage.new( + text, 2025, current_column: 1 + ) + + revenue = page.section_rows(/\ARevenus\b/i, /\ACharges\b/i) + expenses = page.section_rows(/\ACharges\b/i, /\AExc[eé]dent\b/i) + + assert_equal [ [ "Taxes", "5129613" ], [ "Autres revenus", "91608" ] ], + revenue.map { [ _1[:label], _1[:raw_text] ] } + assert_equal [ [ "Administration générale", "1259235" ] ], + expenses.map { [ _1[:label], _1[:raw_text] ] } + end + + test "falls back only for the configured deterministic parser error" do + primary = Object.new + primary.define_singleton_method(:run) { raise Warehouse::FinancialStatementExtraction::QuebecFormPipeline::Unsupported, "layout" } + fallback = Object.new + fallback.define_singleton_method(:run) { :model_result } + pipeline = Warehouse::FinancialStatementExtraction::FallbackPipeline.new( + primary:, fallback:, on: Warehouse::FinancialStatementExtraction::QuebecFormPipeline::Unsupported + ) + + assert_equal :model_result, pipeline.run + end +end diff --git a/test/models/warehouse/financial_statement_extraction/reviewer_test.rb b/test/models/warehouse/financial_statement_extraction/reviewer_test.rb new file mode 100644 index 00000000..67d994ff --- /dev/null +++ b/test/models/warehouse/financial_statement_extraction/reviewer_test.rb @@ -0,0 +1,408 @@ +require "test_helper" + +class Warehouse::FinancialStatementExtraction::ReviewerTest < ActiveSupport::TestCase + setup do + @release = Warehouse::InstitutionRelease.create!( + version: "2026-09-01", effective_on: Date.new(2026, 9, 1), schema_version: "1.0", + published_at: Time.utc(2026, 9, 1), geography_vintage: 2021, attribution: "Test" + ) + source = Warehouse::InstitutionSource.create!( + institution_release: @release, canonical_id: "ca/sources/reviewer-test", + publisher_name: "Test", title_en: "Test", url: "https://example.test/source", + retrieved_at: @release.published_at, languages: [ "en" ] + ) + institution = Warehouse::Institution.create!( + institution_release: @release, institution_source: source, canonical_id: "ca/sk/example", + name_en: "Example", institution_type: "government", government_level: "municipal", status: "active" + ) + document = Warehouse::InstitutionDocument.create!( + institution_release: @release, institution:, institution_source: source, + canonical_id: "ca/sk/example/documents/financial-statements/2025/general", + document_type: "financial-statements", document_variant: "general" + ) + @asset_root = Pathname(Dir.mktmpdir) + sha = Digest::SHA256.hexdigest("source") + relative = Pathname("sha256/#{sha.first(2)}/#{sha}.pdf") + path = @asset_root.join(relative) + path.dirname.mkpath + path.write("source") + asset = Warehouse::InstitutionDocumentAsset.create!( + institution_release: @release, institution_document: document, content_sha256: sha, + asset_role: "final", preferred: true, download_url: "https://example.test/statement.pdf", + retrieved_at: @release.published_at, archive_path: relative.to_s, + mime_type: "application/pdf", byte_size: path.size, rights_status: "metadata_only" + ) + @extraction = Warehouse::FinancialStatementExtraction.create!( + institution_release: @release, institution_canonical_id: institution.canonical_id, + document_canonical_id: document.canonical_id, asset_sha256: asset.content_sha256, + fiscal_year_end: Date.new(2025, 12, 31), extractor_version: "detailed-psas-v1", + status: "extracted", check_results: [ + { id: "source_identity", status: "pass", detail: "source hash matches" } + ], llm_response_snapshot: { "headline" => { + "remeasurement_present" => false, "operations_adjustment_present" => false, + "rollforward_adjustment_present" => false + } } + ) + create_fact("total_financial_assets", 100, "financial_position") + create_fact("total_liabilities", 60, "financial_position") + create_fact("net_financial_assets", 40, "financial_position") + create_fact("total_non_financial_assets", 160, "financial_position") + create_fact("accumulated_surplus", 200, "financial_position") + create_fact("total_revenue", 80, "operations") + create_fact("total_expenses", 70, "operations") + create_fact("annual_surplus", 10, "operations") + create_line_item("revenue", "Revenue item", 80) + create_line_item("expense", "Expense item", 70) + end + + teardown { FileUtils.remove_entry(@asset_root) } + + test "independently revalidates source evidence and approves a detailed extraction" do + result = Warehouse::FinancialStatementExtraction::Reviewer.new( + extraction: @extraction, asset_root: @asset_root, page_locator: locator + ).review! + + assert_equal "approved", result.status + assert_equal "approved", @extraction.reload.status + assert_equal "deterministic-source-reaudit-v1", @extraction.reviewed_by + end + + test "retries a result parked for review after deterministic checks improve" do + @extraction.update!(status: "needs_review") + + result = Warehouse::FinancialStatementExtraction::Reviewer.new( + extraction: @extraction, asset_root: @asset_root, page_locator: locator + ).review! + + assert_equal "approved", result.status + assert_equal "approved", @extraction.reload.status + end + + test "does not overwrite an approval committed while a slower review is auditing" do + reviewer = Warehouse::FinancialStatementExtraction::Reviewer.new( + extraction: @extraction, asset_root: @asset_root, page_locator: locator + ) + result = Warehouse::FinancialStatementExtraction::Reviewer::Result.new( + status: "needs_review", + checks: [ { id: "late_audit", status: "fail", detail: "stale result" } ] + ) + reviewer.stub(:audit, -> { + Warehouse::FinancialStatementExtraction.where(id: @extraction.id).update_all( + status: "approved", reviewed_by: "deterministic-source-reaudit-v1", + reviewed_at: Time.current, review_notes: "concurrent approval" + ) + result + }) do + assert_equal result, reviewer.review! + end + + @extraction.reload + assert_equal "approved", @extraction.status + assert_equal "deterministic-source-reaudit-v1", @extraction.reviewed_by + assert_equal "concurrent approval", @extraction.review_notes + refute_equal result.checks.map(&:stringify_keys), @extraction.check_results + end + + test "audits an already approved extraction without mutating its review provenance" do + @extraction.update!(check_results: [ + { id: "legacy_source_identity", status: "pass", detail: "legacy source check passed" } + ]) + @extraction.approve!(reviewer: "legacy-reviewer") + + result = Warehouse::FinancialStatementExtraction::Reviewer.new( + extraction: @extraction, asset_root: @asset_root, page_locator: locator + ).audit + + assert_equal "approved", result.status + assert_equal "legacy-reviewer", @extraction.reload.reviewed_by + end + + test "promotes a passing legacy approval while preserving prior provenance on the row" do + legacy_checks = [ + { id: "legacy_source_identity", status: "pass", detail: "legacy source check passed" } + ] + @extraction.update!(check_results: legacy_checks) + @extraction.approve!(reviewer: "legacy-reviewer") + previous_reviewed_at = @extraction.reviewed_at + previous_digest = Digest::SHA256.hexdigest(JSON.generate(legacy_checks)) + + result = Warehouse::FinancialStatementExtraction::Reviewer.new( + extraction: @extraction, asset_root: @asset_root, page_locator: locator + ).reaudit! + + assert_equal "approved", result.status + assert_equal "deterministic-source-reaudit-v1", @extraction.reload.reviewed_by + assert_operator @extraction.reviewed_at, :>, previous_reviewed_at + assert_includes @extraction.review_notes, "previous reviewer=legacy-reviewer" + assert_includes @extraction.review_notes, previous_digest + assert_equal result.checks.map(&:stringify_keys), @extraction.check_results + end + + test "does not mutate a legacy approval when its re-audit does not pass" do + @extraction.financial_statement_facts.update_all(scale: 1000) + @extraction.financial_statement_line_items.update_all(scale: 1000) + @extraction.approve!(reviewer: "legacy-reviewer") + before = @extraction.attributes.deep_dup + + result = Warehouse::FinancialStatementExtraction::Reviewer.new( + extraction: @extraction, asset_root: @asset_root, page_locator: locator + ).reaudit! + + assert_equal "needs_review", result.status + assert_equal before, @extraction.reload.attributes + end + + test "does not mutate a legacy approval when its re-audit raises" do + @extraction.approve!(reviewer: "legacy-reviewer") + before = @extraction.attributes.deep_dup + broken_locator = Object.new + broken_locator.define_singleton_method(:locate) { raise "broken source reader" } + reviewer = Warehouse::FinancialStatementExtraction::Reviewer.new( + extraction: @extraction, asset_root: @asset_root, page_locator: broken_locator + ) + + assert_raises(RuntimeError) { reviewer.reaudit! } + assert_equal before, @extraction.reload.attributes + end + + test "promotes a visually verified legacy approval with visual provenance" do + @extraction.approve!(reviewer: "legacy-reviewer") + response = { "claims" => [ { + "id" => "evidence:total_expenses", "found" => true, + "transcribed_label" => "Total expenses", "transcribed_category" => "", + "raw_text" => "70", "column_year" => "Actual 2025", "excerpt_page" => 1 + } ] } + + result = Warehouse::FinancialStatementExtraction::Reviewer.new( + extraction: @extraction, asset_root: @asset_root, page_locator: visually_failing_locator, + visual_llm_client: ->(**) { response } + ).reaudit! + + assert_equal "approved", result.status + assert_equal "deterministic-plus-visual-reaudit-v1", @extraction.reload.reviewed_by + assert_includes @extraction.review_notes, "blind visual source transcription also matched" + end + + test "independently reviews prairie parser evidence using table OCR text" do + @extraction.update!(llm_response_snapshot: @extraction.llm_response_snapshot.merge( + "parser" => Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline::PARSER_VERSION + )) + base = locator.locate + table_locator = Struct.new(:result, :calls) do + def locate = result + def ocr_table_page(page) + calls << page + result.page_texts.fetch(page) + end + end.new(base, []) + + result = Warehouse::FinancialStatementExtraction::Reviewer.new( + extraction: @extraction, asset_root: @asset_root, page_locator: table_locator + ).review! + + assert_equal "approved", result.status + assert_equal [ 1 ], table_locator.calls + end + + test "retains independent table OCR review support for prairie parser v2" do + @extraction.update!(llm_response_snapshot: @extraction.llm_response_snapshot.merge( + "parser" => "prairie-municipal-form-v2" + )) + base = locator.locate + table_locator = Struct.new(:result, :calls) do + def locate = result + def ocr_table_page(page) + calls << page + result.page_texts.fetch(page) + end + end.new(base, []) + reviewer = Warehouse::FinancialStatementExtraction::Reviewer.new( + extraction: @extraction, asset_root: @asset_root, page_locator: table_locator + ) + + assert_equal 8, reviewer.send(:review_ocr_page_limit) + reviewer.send(:enrich_parser_table_ocr, base, table_locator) + assert_equal [ 1 ], table_locator.calls + end + + test "independently reviews cited Quebec form pages using saved OCR provenance" do + @extraction.update!(llm_response_snapshot: @extraction.llm_response_snapshot.merge( + "parser" => Warehouse::FinancialStatementExtraction::QuebecFormPipeline::PARSER_VERSION, + "details" => { "ocr_pages" => [ 1 ] } + )) + base = locator.locate + table_locator = Struct.new(:result, :calls) do + def locate = result + def ocr_table_page(page) + calls << page + result.page_texts.fetch(page) + end + end.new(base, []) + + result = Warehouse::FinancialStatementExtraction::Reviewer.new( + extraction: @extraction, asset_root: @asset_root, page_locator: table_locator + ).review! + + assert_equal "approved", result.status + assert_equal [ 1 ], table_locator.calls + end + + test "table OCRs unique valid generic evidence pages with a hard cap" do + last_item = nil + 13.times do |index| + last_item = create_line_item("expense", "Extra #{index}", index + 1) + last_item.update!(source_page: index + 2) + end + last_item.update!(source_page: 99) + pages = (1..14).index_with { "source" } + located = Warehouse::FinancialStatementExtraction::PageLocator::Result.new( + page_count: 14, page_texts: pages, position_page: 1, + operations_page: 1, candidate_pages: [ 1 ], ocr_pages: [] + ) + table_locator = Struct.new(:result, :calls) do + def ocr_table_page(page) + calls << page + result.page_texts.fetch(page) + end + end.new(located, []) + reviewer = Warehouse::FinancialStatementExtraction::Reviewer.new( + extraction: @extraction, asset_root: @asset_root, page_locator: table_locator + ) + + reviewer.send(:enrich_parser_table_ocr, located, table_locator) + + assert_equal (1..12).to_a, table_locator.calls + refute_includes table_locator.calls, 99 + end + + test "blind visual transcription can clear the only deterministic evidence failure" do + visual_calls = [] + response = { "claims" => [ { + "id" => "evidence:total_expenses", "found" => true, + "transcribed_label" => "Total expenses", "transcribed_category" => "", + "raw_text" => "70", "column_year" => "Actual 2025", "excerpt_page" => 1 + } ] } + result = Warehouse::FinancialStatementExtraction::Reviewer.new( + extraction: @extraction, asset_root: @asset_root, page_locator: visually_failing_locator, + visual_llm_client: ->(prompt:, pdf_path:) do + visual_calls << [ prompt, pdf_path ] + response + end + ).review! + + assert_equal "approved", result.status + assert_equal "deterministic-plus-visual-reaudit-v1", @extraction.reload.reviewed_by + assert_equal 1, visual_calls.length + refute_includes visual_calls.first.first, "raw_text: 70" + assert_equal "pass", result.checks.find { _1[:id] == "visual_evidence:evidence:total_expenses" }[:status] + end + + test "visual verifier refuses mismatched values" do + checks = [ { id: "evidence:total_expenses", status: "fail", detail: "OCR miss" } ] + response = { "claims" => [ { + "id" => "evidence:total_expenses", "found" => true, + "transcribed_label" => "Total expenses", "transcribed_category" => "", + "raw_text" => "71", "column_year" => "Actual 2025", "excerpt_page" => 1 + } ] } + result = Warehouse::FinancialStatementExtraction::VisualEvidenceReviewer.new( + extraction: @extraction, page_locator: visually_failing_locator, + llm_client: ->(**) { response } + ).apply(checks) + + assert_equal "fail", result.find { _1[:id] == "evidence:total_expenses" }[:status] + visual = result.find { _1[:id] == "visual_evidence:evidence:total_expenses" } + assert_equal "fail", visual[:status] + assert_includes visual[:detail], "value mismatch" + end + + test "visual verifier is gated off by any non-evidence failure" do + called = false + checks = [ + { id: "evidence:total_expenses", status: "fail", detail: "OCR miss" }, + { id: "line_sum:expense", status: "fail", detail: "incomplete" } + ] + result = Warehouse::FinancialStatementExtraction::VisualEvidenceReviewer.new( + extraction: @extraction, page_locator: visually_failing_locator, + llm_client: ->(**) { called = true } + ).apply(checks) + + refute called + assert_equal checks, result + end + + test "visual verifier uses sorted physical pages and rejects incomplete responses" do + @extraction.financial_statement_facts.find_by!(concept: "total_expenses").update!(source_page: 2) + expense = @extraction.financial_statement_line_items.find_by!(flow: "expense") + checks = [ + { id: "evidence:total_expenses", status: "fail", detail: "OCR miss" }, + { id: "line_evidence:expense-items-expense-item", status: "fail", detail: "OCR miss" } + ] + pages_seen = [] + excerpt_locator = Struct.new(:pages_seen) do + def with_excerpt(pages) + pages_seen << pages + yield Pathname("visual.pdf") + end + end.new(pages_seen) + result = Warehouse::FinancialStatementExtraction::VisualEvidenceReviewer.new( + extraction: @extraction, page_locator: excerpt_locator, + llm_client: ->(**) do + { "claims" => [ { + "id" => "evidence:total_expenses", "found" => true, + "transcribed_label" => "Total expenses", "transcribed_category" => "", + "raw_text" => "70", "column_year" => "2025", "excerpt_page" => 2 + } ] } + end + ).apply(checks) + + assert_equal [ [ 1, 2 ] ], pages_seen + assert_equal "fail", result.find { _1[:id] == "visual_evidence:verifier" }[:status] + assert_equal 1, expense.source_page + end + + private + + def locator + text = (@extraction.financial_statement_facts.map { "#{_1.raw_label} #{_1.raw_text}" } + + @extraction.financial_statement_line_items.map { "#{_1.label} #{_1.raw_text}" }).join(" ") + located = Warehouse::FinancialStatementExtraction::PageLocator::Result.new( + page_count: 1, page_texts: { 1 => text }, position_page: 1, + operations_page: 1, candidate_pages: [ 1 ], ocr_pages: [] + ) + Struct.new(:result) do + def locate = result + def ocr_table_page(page) = result.page_texts.fetch(page) + end.new(located) + end + + def visually_failing_locator + text = (@extraction.financial_statement_facts.map { "#{_1.raw_label} #{_1.raw_text}" } + + @extraction.financial_statement_line_items.map { "#{_1.label} #{_1.raw_text}" }).join(" ") + .sub("Total expenses 70", "Expenses 70") + located = Warehouse::FinancialStatementExtraction::PageLocator::Result.new( + page_count: 1, page_texts: { 1 => text }, position_page: 1, + operations_page: 1, candidate_pages: [ 1 ], ocr_pages: [] + ) + Struct.new(:result) do + def locate = result + def ocr_table_page(page) = result.page_texts.fetch(page) + def with_excerpt(_pages) = yield(Pathname("visual.pdf")) + end.new(located) + end + + def create_fact(concept, value, statement) + @extraction.financial_statement_facts.create!( + concept:, value:, raw_text: value.to_s, raw_label: concept.humanize, + scale: 1, statement:, source_page: 1, column_year: "2025", extraction_confidence: 0.99 + ) + end + + def create_line_item(flow, label, value) + position = @extraction.financial_statement_line_items.where(flow:).maximum(:position).to_i + position += 1 if @extraction.financial_statement_line_items.where(flow:).exists? + @extraction.financial_statement_line_items.create!( + flow:, category: "Items", label:, value:, raw_text: value.to_s, + scale: 1, source_page: 1, column_year: "2025", position:, extraction_confidence: 0.99 + ) + end +end diff --git a/test/models/warehouse/financial_statement_extraction/saskatchewan_form_pipeline_test.rb b/test/models/warehouse/financial_statement_extraction/saskatchewan_form_pipeline_test.rb new file mode 100644 index 00000000..33a44e7c --- /dev/null +++ b/test/models/warehouse/financial_statement_extraction/saskatchewan_form_pipeline_test.rb @@ -0,0 +1,624 @@ +require "test_helper" + +class Warehouse::FinancialStatementExtraction::SaskatchewanFormPipelineTest < ActiveSupport::TestCase + class TableOcrLocator + attr_reader :calls + + def initialize(result, table_texts) + @result = result + @table_texts = table_texts + @calls = [] + end + + def locate = @result + + def ocr_table_page(page) + calls << page + @table_texts.fetch(page) + end + end + + test "reads the actual column without treating label hyphens as blank cells" do + page = Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline::EnglishPage.new(<<~TEXT, fiscal_year: 2022, kind: :operations) + 2022 Budget 2022 2021 + REVENUES (unaudited) + Taxes and Other Unconditional Revenue (Schedule 1) 180,880 211,152 185,902 + Tangible Capital Asset Sales - Gain (Schedule 4, 5) - 122 400 + Environmental Services 30.380 32,228 30.347 + Protective Services 13,230 1 1,375 13,319 + Total Revenues 180,880 211,274 186,302 + EXPENSES + General Government Services (Schedule 3) 62,280 45,222 55,107 + Total Expenses 62,280 45,222 55,107 + Annual Surplus = 97,365 55,340 + TEXT + + assert_equal "211,152", page.lines.find { _1[:label].start_with?("Taxes") }.fetch(:current) + capital_sale = page.lines.find { _1[:label].start_with?("Tangible") } + assert_equal "Tangible Capital Asset Sales - Gain (Schedule 4, 5)", capital_sale.fetch(:label) + assert_equal "122", capital_sale.fetch(:current) + assert_equal "32,228", page.lines.find { _1[:label].start_with?("Environmental") }.fetch(:current) + assert_equal "1 1,375", page.lines.find { _1[:label].start_with?("Protective") }.fetch(:current) + assert_equal "97,365", page.lines.find { _1[:label].start_with?("Annual Surplus") }.fetch(:current) + assert_equal 4, page.between(/\AREVENUES/, /\ATotal Revenues/).length + end + + test "drops only an unambiguous trailing OCR null column" do + page = Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline::EnglishPage.new(<<~TEXT, fiscal_year: 2025, kind: :position) + 2025 2024 + ACCUMULATED SURPLUS 15,056,773 14,004,009 - + Legitimate prior-year null 100 - + Adjustment 2025 100 - + Ambiguous extra null 100 - - + TEXT + + assert_equal "15,056,773", page.lines.find { _1[:label].start_with?("ACCUMULATED") }.fetch(:current) + assert_equal "100", page.lines.find { _1[:label].start_with?("Legitimate") }.fetch(:current) + assert_equal "100", page.lines.find { _1[:label].start_with?("Adjustment") }.fetch(:current) + assert_equal "-", page.lines.find { _1[:label].start_with?("Ambiguous") }.fetch(:current) + end + + test "recognizes surplus of revenues over expenditures" do + page = Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline::EnglishPage.new(<<~TEXT, fiscal_year: 2025, kind: :operations) + 2025 Budget 2025 2024 + Surplus (deficit) of revenues over expenditures 10 20 15 + TEXT + row = page.lines.sole + + assert_match Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline::ANNUAL_SURPLUS_PATTERN, + row.fetch(:label) + assert_equal "20", row.fetch(:current) + end + + test "trims only an adjacent truncated prior-year OCR fragment" do + page = Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline::EnglishPage.new(<<~TEXT, fiscal_year: 2025, kind: :operations) + 2025 Budget 2025 2024 + Comma-truncated prior 100 200 222,23 + Dot-truncated prior 100 200 1.641,28 + Genuine small prior 100 200 23 + Note reference (Note 13) 100 200 23 + Truncated current 100 2,00 300 + TEXT + + assert_equal "200", page.lines.find { _1[:label].start_with?("Comma") }.fetch(:current) + assert_equal "200", page.lines.find { _1[:label].start_with?("Dot") }.fetch(:current) + assert_equal "200", page.lines.find { _1[:label].start_with?("Genuine") }.fetch(:current) + assert_equal "200", page.lines.find { _1[:label].start_with?("Note") }.fetch(:current) + assert_equal "00", page.lines.find { _1[:label].start_with?("Truncated current") }.fetch(:current) + end + + test "keeps unlabeled OCR total rows available for form-level association" do + page = Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline::EnglishPage.new(<<~TEXT, fiscal_year: 2022, kind: :position) + 2022 + 2021 + Total Financial Assets + Cash 300,000 290,000 + 446,415 432,511 + TEXT + + total = page.fetch(/\ATotal Financial Assets/) + assert_nil total.fetch(:current) + assert_equal "446,415", page.lines.find { _1[:label].blank? }.fetch(:current) + end + + test "does not mistake an individual liability for total liabilities" do + page = Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline::EnglishPage.new(<<~TEXT, fiscal_year: 2023, kind: :position) + 2023 2022 + FINANCIAL LIABILITIES + Liability for Contaminated Sites (Note 13) - - + Total Liabilities 64,241 73,211 + TEXT + + total = page.fetch(Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline::FACT_LABELS.fetch(:total_liabilities)) + + assert_equal "Total Liabilities", total.fetch(:label) + assert_equal "64,241", total.fetch(:current) + end + + test "associates bounded unlabeled position totals after multiple component rows" do + page = Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline::EnglishPage.new(<<~TEXT, fiscal_year: 2023, kind: :position) + 2023 2022 + FINANCIAL ASSETS + Cash 100 90 + Investments 200 180 + Receivables 50 50 + ____ 350 320 + LIABILITIES + Accounts payable 100 90 + Long-term debt 50 50 + 150 140 + NET FINANCIAL ASSETS 200 180 + NON-FINANCIAL ASSETS + Tangible capital assets 500 480 + Inventories 20 20 + 520 500 + ACCUMULATED SURPLUS 720 680 + TEXT + pipeline = Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline.allocate + + financial_assets = pipeline.send(:position_total, page, :total_financial_assets) + liabilities = pipeline.send(:position_total, page, :total_liabilities) + non_financial_assets = pipeline.send(:position_total, page, :total_non_financial_assets) + + assert_equal "350", financial_assets.fetch(:current) + assert_equal "150", liabilities.fetch(:current) + assert_equal "520", non_financial_assets.fetch(:current) + assert_equal BigDecimal("0.95"), financial_assets.fetch(:extraction_confidence) + end + + test "uses a sole non-financial asset component as the bounded total" do + page = Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline::EnglishPage.new(<<~TEXT, fiscal_year: 2023, kind: :position) + 2023 2022 + NON-FINANCIAL ASSETS + Tangible capital assets 520 500 + ACCUMULATED SURPLUS 720 680 + TEXT + pipeline = Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline.allocate + + total = pipeline.send(:position_total, page, :total_non_financial_assets) + + assert_equal "Tangible capital assets", total.fetch(:label) + assert_equal "520", total.fetch(:current) + assert_equal BigDecimal("0.90"), total.fetch(:extraction_confidence) + end + + test "rejects an unlabeled position value when the section lacks multiple labeled components" do + page = Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline::EnglishPage.new(<<~TEXT, fiscal_year: 2023, kind: :position) + 2023 2022 + FINANCIAL ASSETS + Cash 100 90 + 100 90 + LIABILITIES + TEXT + pipeline = Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline.allocate + + assert_nil pipeline.send(:position_total, page, :total_financial_assets) + end + + test "infers omitted standardized form headers and joins wrapped annual surplus labels" do + page = Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline::EnglishPage.new(<<~TEXT, fiscal_year: 2018, kind: :operations) + MUNICIPALITY OF EXAMPLE + Statement of Operations + Year Ended December 31, 2018 + + REVENUES + Taxes (Schedule 1) 100,000 110,000 90,000 + Fees and charges (Schedule 4) 20,000 25,000 15,000 + Other revenue 5,000 6,000 4,000 + 125,000 141,000 109,000 + + EXPENSES + General government (Schedule 3) 100,000 100,000 90,000 + Transportation 10,000 10,000 9,000 + Recreation 5,000 5,000 4,000 + 115,000 115,000 103,000 + + EXCESS (DEFICIENCY) OF REVENUES + OVER EXPENSES BEFORE OTHER + CAPITAL CONTRIBUTIONS 10,000 26,000 6,000 + Capital grants 2,000 3,000 1,000 + EXCESS (DEFICIENCY) OF REVENUES OVER + EXPENSES 12,000 29,000 7,000 + TEXT + + taxes = page.lines.find { _1[:label].start_with?("Taxes") } + annual_rows = page.lines.select do |row| + row[:label].match?(Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline::ANNUAL_SURPLUS_PATTERN) + end + + assert_equal "110,000", taxes.fetch(:current) + assert_equal 2, annual_rows.length + assert_equal "29,000", annual_rows.last.fetch(:current) + assert_equal "EXCESS (DEFICIENCY) OF REVENUES OVER EXPENSES", annual_rows.last.fetch(:label) + end + + test "joins shortfall surplus labels and keeps the final row after before-other subtotal" do + page = Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline::EnglishPage.new(<<~TEXT, fiscal_year: 2018, kind: :operations) + 2018 Budget 2018 2017 + REVENUE + Taxes 90 100 80 + Total Revenue 90 100 80 + EXPENSES + Services 80 90 70 + Total Expenses 80 90 70 + EXCESS (SHORTFALL) OF REVENUE OVER + EXPENSES - BEFORE OTHER 10 10 10 + Capital grants - 20 - + EXCESS (SHORTFALL) OF REVENUE OVER + EXPENSES 10 30 10 + TEXT + + rows = page.lines.select do |row| + row[:label].match?(Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline::ANNUAL_SURPLUS_PATTERN) + end + + assert_equal 2, rows.length + assert_equal "10", rows.first.fetch(:current) + assert_equal "30", rows.last.fetch(:current) + assert_equal "EXCESS (SHORTFALL) OF REVENUE OVER EXPENSES", rows.last.fetch(:label) + end + + test "joins a wrapped position total to an unlabeled numeric row" do + page = Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline::EnglishPage.new(<<~TEXT, fiscal_year: 2023, kind: :position) + MUNICIPALITY OF EXAMPLE + Statement of Financial Position + As at December 31, 2023 + + Cash 300,000 290,000 + Investments 100,000 90,000 + Receivables 40,000 35,000 + Total Financial Assets + 440,000 415,000 + TEXT + + total = page.fetch(Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline::FACT_LABELS.fetch(:total_financial_assets)) + + assert_equal "Total Financial Assets", total.fetch(:label) + assert_equal "440,000", total.fetch(:current) + end + + test "handles an optional opening surplus without duplicating capital contributions" do + source = Tempfile.new([ "municipal-statement", ".pdf" ]) + source.write("source") + source.close + sha = Digest::SHA256.file(source.path).hexdigest + position = <<~TEXT + 2025 2024 + Total Financial Assets 100 90 + LIABILITIES + Total Liabi lities + - - + 60 50 + NET FINANCIAL ASSETS 40 40 + Total Non-Financial Assets 160 140 + ACCUMULATED SURPLUS 200 180 + TEXT + operations = <<~TEXT + 2025 Budget 2025 2024 + REVENUES + Taxes 90 90 80 + Provincial Capital Grants and Contributions 10 10 - + 100 100 80 + Total Revenues - - - + EXPENSES + General Government 80 80 60 + Total Expenses 80 80 60 + Annual Surplus 20 20 20 + Annual Surplus - - - + TEXT + located = Warehouse::FinancialStatementExtraction::PageLocator::Result.new( + page_count: 2, page_texts: { 1 => position, 2 => operations }, + position_page: 1, operations_page: 2, candidate_pages: [ 1, 2 ], ocr_pages: [] + ) + locator = Struct.new(:result) { def locate = result }.new(located) + + result = Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline.new( + pdf_path: source.path, institution_canonical_id: "ca/sk/example", + document_canonical_id: "ca/sk/example/documents/financial-statements/2025/general", + asset_sha256: sha, fiscal_year_end: Date.new(2025, 12, 31), page_locator: locator + ).run + + capital_rows = result.line_items.select { _1[:label].include?("Capital Grants") } + assert_equal "extracted", result.status + assert_equal 1, capital_rows.length + assert_equal 60, result.facts.find { _1[:concept] == "total_liabilities" }.fetch(:value) + refute result.facts.any? { _1[:concept] == "opening_accumulated_surplus" } + assert result.checks.find { _1[:id] == "line_sum:revenue" && _1[:status] == "pass" } + assert result.checks.find { _1[:id] == "surplus_rollforward" && _1[:status] == "skip" } + ensure + source&.unlink + end + + test "bounds expense line items by annual surplus when the printed total is unlabeled" do + source = Tempfile.new([ "municipal-statement", ".pdf" ]) + source.write("source") + source.close + sha = Digest::SHA256.file(source.path).hexdigest + position = <<~TEXT + 2017 2016 + Total Financial Assets 100 90 + Total Liabilities 60 50 + NET FINANCIAL ASSETS 40 40 + Total Non-Financial Assets 170 140 + ACCUMULATED SURPLUS 210 180 + TEXT + operations = <<~TEXT + 2017 Budget 2017 2016 + REVENUE + Taxes 100 100 90 + 100 100 90 + EXPENSE + Services 80 80 70 + 80 80 70 + EXCESS (SHORTFALL) OF REVENUE OVER EXPENSES BEFORE OTHER 20 20 20 + OTHER + Government transfers for capital 10 10 10 + EXCESS (SHORTFALL) OF REVENUE OVER EXPENSES 30 30 30 + ACCUMULATED SURPLUS, BEGINNING OF YEAR 180 180 160 + ACCUMULATED SURPLUS, END OF YEAR 210 210 180 + TEXT + located = Warehouse::FinancialStatementExtraction::PageLocator::Result.new( + page_count: 2, page_texts: { 1 => position, 2 => operations }, + position_page: 1, operations_page: 2, candidate_pages: [ 1, 2 ], ocr_pages: [] + ) + locator = Struct.new(:result) do + def locate = result + def ocr_table_page(page) = result.page_texts.fetch(page) + end.new(located) + + result = Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline.new( + pdf_path: source.path, institution_canonical_id: "ca/ab/example", + document_canonical_id: "ca/ab/example/documents/financial-statements/2017/general", + asset_sha256: sha, fiscal_year_end: Date.new(2017, 12, 31), page_locator: locator + ).run + + assert_equal "extracted", result.status + assert_equal [ "Services" ], result.line_items.select { _1[:flow] == "expense" }.pluck(:label) + assert_equal [ "Taxes", "Government transfers for capital" ], + result.line_items.select { _1[:flow] == "revenue" }.pluck(:label) + expense_total = result.facts.find { _1[:concept] == "total_expenses" } + assert_equal 80, expense_total.fetch(:value) + assert_equal "EXPENSE", expense_total.fetch(:raw_label) + assert_equal BigDecimal("0.95"), expense_total.fetch(:extraction_confidence) + assert_includes result.response.dig("details", "position_total_fallbacks"), { + "concept" => "total_expenses", "type" => "unlabeled_section_total" + } + assert result.checks.find { _1[:id] == "line_sum:expense" && _1[:status] == "pass" } + ensure + source&.unlink + end + + test "retries shaded total bands with thresholded table OCR" do + source = Tempfile.new([ "municipal-statement", ".pdf" ]) + source.write("source") + source.close + sha = Digest::SHA256.file(source.path).hexdigest + plain_position = <<~TEXT + Statement of Financial Position 2025 2024 + Total Financial Assets + Total Liabilities + NET FINANCIAL ASSETS + Total Non-Financial Assets + ACCUMULATED SURPLUS + 100 90 + 60 50 + 40 40 + 160 140 + 200 180 + TEXT + plain_operations = <<~TEXT + Statement of Operations 2025 Budget 2025 2024 + REVENUES + Taxes + Total Revenues + EXPENSES + Services + Total Expenses + Annual Surplus + 100 100 90 + 80 80 70 + 20 20 20 + TEXT + ocr_position = <<~TEXT + Statement of Financial Position + 2025 + 2024 + Total Financial Assets 100 90 + Total Liabilities 60 50 + NET FINANCIAL ASSETS 40 40 + Total Non-Financial Assets 160 140 + ACCUMULATED SURPLUS 200 180 + TEXT + ocr_operations = <<~TEXT + Statement of Operations + 2025 Budget 2025 2024 + REVENUES + Taxes 100 100 90 + Total Revenues 100 100 90 + EXPENSES + Services 80 80 70 + Total Expenses 80 80 70 + Surplus (Deficit) of Revenues over Expenses 20 20 20 + Accumulated Surplus, Beginning of Year 180 180 160 + TEXT + located = Warehouse::FinancialStatementExtraction::PageLocator::Result.new( + page_count: 2, page_texts: { 1 => plain_position, 2 => plain_operations }, + position_page: 1, operations_page: 2, candidate_pages: [ 1, 2 ], ocr_pages: [] + ) + locator = TableOcrLocator.new(located, { 1 => ocr_position, 2 => ocr_operations }) + + result = Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline.new( + pdf_path: source.path, institution_canonical_id: "ca/sk/example", + document_canonical_id: "ca/sk/example/documents/financial-statements/2025/general", + asset_sha256: sha, fiscal_year_end: Date.new(2025, 12, 31), page_locator: locator + ).run + + assert_equal "extracted", result.status + assert_equal [ 1, 2 ], locator.calls + assert_equal [ 1, 2 ], result.locator_result.ocr_pages + assert_equal 100, result.facts.find { _1[:concept] == "total_financial_assets" }.fetch(:value) + ensure + source&.unlink + end + + test "keeps original position text when table OCR is also incomplete" do + plain_position = <<~TEXT + Statement of Financial Position 2025 2024 + Total Financial Assets + Total Liabilities + NET FINANCIAL ASSETS + Total Non-Financial Assets + ACCUMULATED SURPLUS + TEXT + operations = <<~TEXT + Statement of Operations 2025 Budget 2025 2024 + Total Revenues 100 100 90 + Total Expenses 80 80 70 + Annual Surplus 20 20 20 + TEXT + located = Warehouse::FinancialStatementExtraction::PageLocator::Result.new( + page_count: 2, page_texts: { 1 => plain_position, 2 => operations }, + position_page: 1, operations_page: 2, candidate_pages: [ 1, 2 ], ocr_pages: [] + ) + locator = TableOcrLocator.new(located, { 1 => "Statement of Financial Position 2025 2024\nCash 100 90" }) + pipeline = Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline.new( + pdf_path: "unused.pdf", institution_canonical_id: "ca/sk/example", + document_canonical_id: "ca/sk/example/documents/financial-statements/2025/general", + asset_sha256: "a" * 64, fiscal_year_end: Date.new(2025, 12, 31), page_locator: locator + ) + + enriched = pipeline.send(:enrich_with_table_ocr, located) + + assert_equal [ 1 ], locator.calls + assert_equal plain_position, enriched.page_texts.fetch(1) + assert_empty enriched.ocr_pages + end + + test "keeps original operations text when table OCR is also incomplete" do + position = <<~TEXT + Statement of Financial Position 2025 2024 + Total Financial Assets 100 90 + Total Liabilities 60 50 + NET FINANCIAL ASSETS 40 40 + Total Non-Financial Assets 160 140 + ACCUMULATED SURPLUS 200 180 + TEXT + operations = <<~TEXT + Statement of Operations 2025 Budget 2025 2024 + REVENUES + Total Revenues + EXPENSES + Total Expenses + Annual Surplus + TEXT + located = Warehouse::FinancialStatementExtraction::PageLocator::Result.new( + page_count: 2, page_texts: { 1 => position, 2 => operations }, + position_page: 1, operations_page: 2, candidate_pages: [ 1, 2 ], ocr_pages: [] + ) + locator = TableOcrLocator.new(located, { + 2 => "Statement of Operations 2025 Budget 2025 2024\nTotal Revenues 100 100 90" + }) + pipeline = Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline.new( + pdf_path: "unused.pdf", institution_canonical_id: "ca/sk/example", + document_canonical_id: "ca/sk/example/documents/financial-statements/2025/general", + asset_sha256: "a" * 64, fiscal_year_end: Date.new(2025, 12, 31), page_locator: locator + ) + + enriched = pipeline.send(:enrich_with_table_ocr, located) + + assert_equal [ 2 ], locator.calls + assert_equal operations, enriched.page_texts.fetch(2) + assert_empty enriched.ocr_pages + end + + test "retries malformed operations values with table OCR exactly once" do + source = Tempfile.new([ "municipal-statement", ".pdf" ]) + source.write("source") + source.close + sha = Digest::SHA256.file(source.path).hexdigest + position = <<~TEXT + 2025 2024 + Total Financial Assets 100 90 + Total Liabilities 60 50 + NET FINANCIAL ASSETS 40 40 + Total Non-Financial Assets 160 140 + ACCUMULATED SURPLUS 200 180 + TEXT + malformed_operations = <<~TEXT + 2025 Budget 2025 2024 + REVENUES + Taxes 100 100 90 + Total Revenues 999 999 90 + EXPENSES + Services 80 80 70 + Total Expenses 80 80 70 + Annual Surplus 20 20 20 + Accumulated Surplus, Beginning of Year 180 180 160 + TEXT + corrected_operations = malformed_operations.sub("999 999 90", "100 100 90") + located = Warehouse::FinancialStatementExtraction::PageLocator::Result.new( + page_count: 2, page_texts: { 1 => position, 2 => malformed_operations }, + position_page: 1, operations_page: 2, candidate_pages: [ 1, 2 ], ocr_pages: [] + ) + locator = TableOcrLocator.new(located, { 2 => corrected_operations }) + + result = Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline.new( + pdf_path: source.path, institution_canonical_id: "ca/ab/example", + document_canonical_id: "ca/ab/example/documents/financial-statements/2025/general", + asset_sha256: sha, fiscal_year_end: Date.new(2025, 12, 31), page_locator: locator + ).run + + assert_equal "extracted", result.status + assert_equal [ 2 ], locator.calls + assert_equal [ 2 ], result.locator_result.ocr_pages + assert result.checks.find { _1[:id] == "operations_surplus" && _1[:status] == "pass" } + ensure + source&.unlink + end + + test "does not retry an operations page already sourced from OCR" do + source = Tempfile.new([ "municipal-statement", ".pdf" ]) + source.write("source") + source.close + sha = Digest::SHA256.file(source.path).hexdigest + position = <<~TEXT + 2025 2024 + Total Financial Assets 100 90 + Total Liabilities 60 50 + NET FINANCIAL ASSETS 40 40 + Total Non-Financial Assets 160 140 + ACCUMULATED SURPLUS 200 180 + TEXT + operations = <<~TEXT + 2025 Budget 2025 2024 + REVENUES + Taxes 100 100 90 + Total Revenues 999 999 90 + EXPENSES + Services 80 80 70 + Total Expenses 80 80 70 + Annual Surplus 20 20 20 + Accumulated Surplus, Beginning of Year 180 180 160 + TEXT + located = Warehouse::FinancialStatementExtraction::PageLocator::Result.new( + page_count: 2, page_texts: { 1 => position, 2 => operations }, + position_page: 1, operations_page: 2, candidate_pages: [ 1, 2 ], ocr_pages: [ 2 ] + ) + locator = TableOcrLocator.new(located, {}) + pipeline = Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline.new( + pdf_path: source.path, institution_canonical_id: "ca/ab/example", + document_canonical_id: "ca/ab/example/documents/financial-statements/2025/general", + asset_sha256: sha, fiscal_year_end: Date.new(2025, 12, 31), page_locator: locator + ) + + assert_raises(Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline::Unsupported) do + pipeline.run + end + assert_empty locator.calls + ensure + source&.unlink + end + + test "does not OCR after a source hash failure" do + source = Tempfile.new([ "municipal-statement", ".pdf" ]) + source.write("source") + source.close + located = Warehouse::FinancialStatementExtraction::PageLocator::Result.new( + page_count: 1, page_texts: { 1 => "Statement of Operations" }, + position_page: 1, operations_page: 1, candidate_pages: [ 1 ], ocr_pages: [] + ) + locator = TableOcrLocator.new(located, {}) + pipeline = Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline.new( + pdf_path: source.path, institution_canonical_id: "ca/ab/example", + document_canonical_id: "ca/ab/example/documents/financial-statements/2025/general", + asset_sha256: "0" * 64, fiscal_year_end: Date.new(2025, 12, 31), page_locator: locator + ) + + assert_raises(Warehouse::FinancialStatementExtraction::SaskatchewanFormPipeline::Unsupported) do + pipeline.run + end + assert_empty locator.calls + ensure + source&.unlink + end +end diff --git a/test/models/warehouse/financial_statement_extraction/scale_detector_test.rb b/test/models/warehouse/financial_statement_extraction/scale_detector_test.rb new file mode 100644 index 00000000..79c0e3e4 --- /dev/null +++ b/test/models/warehouse/financial_statement_extraction/scale_detector_test.rb @@ -0,0 +1,19 @@ +require "test_helper" + +class Warehouse::FinancialStatementExtraction::ScaleDetectorTest < ActiveSupport::TestCase + test "detects explicit statement units" do + detector = Warehouse::FinancialStatementExtraction::ScaleDetector + + assert_equal 1, detector.detect([ "Statement of Operations ($)" ]) + assert_equal 1_000, detector.detect([ "Statement of Operations (in thousands)" ]) + assert_equal 1_000, detector.detect([ "Amounts in $000's" ]) + assert_equal 1_000, detector.detect([ "Montants en milliers de dollars" ]) + assert_equal 1_000_000, detector.detect([ "CAD in millions" ]) + end + + test "prefers the statement's explicit units over a nearby chart's units" do + text = "For the year ended December 31 (in thousands of dollars)\nIn millions" + + assert_equal 1_000, Warehouse::FinancialStatementExtraction::ScaleDetector.detect([ text ]) + end +end diff --git a/test/models/warehouse/financial_statement_extraction/stored_headline_pipeline_test.rb b/test/models/warehouse/financial_statement_extraction/stored_headline_pipeline_test.rb new file mode 100644 index 00000000..db72a2e7 --- /dev/null +++ b/test/models/warehouse/financial_statement_extraction/stored_headline_pipeline_test.rb @@ -0,0 +1,65 @@ +require "test_helper" + +class Warehouse::FinancialStatementExtraction::StoredHeadlinePipelineTest < ActiveSupport::TestCase + test "reuses an extracted headline without requiring premature approval" do + file = Tempfile.new([ "headline", ".pdf" ]) + file.write("source") + file.flush + extraction = Struct.new( + :status, :extractor_version, :asset_sha256, :financial_statement_facts, + :check_results, :llm_prompt_snapshot, :llm_response_snapshot, + :language, :statement_basis + ).new( + "extracted", Warehouse::FinancialStatementExtraction::Pipeline::EXTRACTOR_VERSION, + Digest::SHA256.file(file.path).hexdigest, [], [], { "prompt" => "headline" }, {}, + "en", "consolidated" + ) + locator_result = Warehouse::FinancialStatementExtraction::PageLocator::Result.new( + page_count: 1, page_texts: { 1 => "statement" }, position_page: 1, + operations_page: 1, candidate_pages: [ 1 ], ocr_pages: [] + ) + locator = Struct.new(:result) { def locate = result }.new(locator_result) + + result = Warehouse::FinancialStatementExtraction::StoredHeadlinePipeline.new( + extraction:, pdf_path: file.path, page_locator: locator + ).run + + assert_equal "extracted", result.status + assert_equal "headline", result.prompt + ensure + file&.close! + end + + test "requires explicit opt in to reuse a headline needing review" do + file = Tempfile.new([ "headline", ".pdf" ]) + file.write("source") + file.flush + extraction = Struct.new( + :status, :extractor_version, :asset_sha256, :financial_statement_facts, + :check_results, :llm_prompt_snapshot, :llm_response_snapshot, + :language, :statement_basis + ).new( + "needs_review", Warehouse::FinancialStatementExtraction::Pipeline::EXTRACTOR_VERSION, + Digest::SHA256.file(file.path).hexdigest, [], [], { "prompt" => "headline" }, {}, + "en", "consolidated" + ) + locator_result = Warehouse::FinancialStatementExtraction::PageLocator::Result.new( + page_count: 1, page_texts: { 1 => "statement" }, position_page: 1, + operations_page: 1, candidate_pages: [ 1 ], ocr_pages: [] + ) + locator = Struct.new(:result) { def locate = result }.new(locator_result) + + assert_raises(ArgumentError) do + Warehouse::FinancialStatementExtraction::StoredHeadlinePipeline.new( + extraction:, pdf_path: file.path, page_locator: locator + ).run + end + result = Warehouse::FinancialStatementExtraction::StoredHeadlinePipeline.new( + extraction:, pdf_path: file.path, page_locator: locator, allow_needs_review: true + ).run + + assert_equal "needs_review", result.status + ensure + file&.close! + end +end diff --git a/test/models/warehouse/financial_statement_extraction/validator_test.rb b/test/models/warehouse/financial_statement_extraction/validator_test.rb index 9d559ad6..dc4c0d3a 100644 --- a/test/models/warehouse/financial_statement_extraction/validator_test.rb +++ b/test/models/warehouse/financial_statement_extraction/validator_test.rb @@ -1,6 +1,36 @@ require "test_helper" class Warehouse::FinancialStatementExtraction::ValidatorTest < ActiveSupport::TestCase + test "accepts other comprehensive loss as rollforward evidence" do + texts = page_texts.merge( + 2 => page_texts.fetch(2) + " Subsidiary operations - other comprehensive (loss) income" + ) + validator = Validator.new( + facts: clean_facts, fiscal_year: 2025, page_texts: texts, + flags: { rollforward_adjustment_present: true } + ) + + check = validator.validate.find { |row| row[:id] == "exception_evidence:rollforward_adjustment_present" } + + assert_equal "pass", check[:status] + end + + test "ignores a declared rollforward adjustment when no opening balance was extracted" do + facts = clean_facts.reject { _1[:concept] == "opening_accumulated_surplus" } + validator = Validator.new( + facts:, fiscal_year: 2025, page_texts: page_texts, + flags: { rollforward_adjustment_present: true } + ) + checks = validator.validate + flag_check = checks.find do |check| + check[:id] == "exception_evidence:rollforward_adjustment_present" + end + + assert validator.acceptable?(checks) + assert_equal "pass", flag_check.fetch(:status) + assert_equal "inactive", flag_check.fetch(:detail) + end + Validator = Warehouse::FinancialStatementExtraction::Validator test "accepts a complete internally consistent extraction" do @@ -57,6 +87,105 @@ class Warehouse::FinancialStatementExtraction::ValidatorTest < ActiveSupport::Te assert_equal "skip", checks.find { |check| check[:id] == "operations_surplus" }.fetch(:status) end + test "accepts source-backed single components when the position identity passes" do + validator = Validator.new( + facts: clean_facts, fiscal_year: 2025, page_texts: page_texts, + flags: { single_component_concepts: %w[total_financial_assets total_non_financial_assets] } + ) + checks = validator.validate + + assert validator.acceptable?(checks) + assert_equal "pass", checks.find { _1[:id] == "position_single_component" }.fetch(:status) + end + + test "rejects single components when remeasurement skips the position surplus identity" do + facts = clean_facts.map(&:dup) + accumulated = facts.find { _1[:concept] == "accumulated_surplus" } + accumulated[:raw_text] = "201,000" + accumulated[:value] = BigDecimal("201000000") + texts = page_texts.merge( + 1 => page_texts.fetch(1).sub("200,000", "201,000") + " Accumulated remeasurement gains" + ) + validator = Validator.new( + facts:, fiscal_year: 2025, page_texts: texts, + flags: { + remeasurement_present: true, + single_component_concepts: %w[total_financial_assets total_non_financial_assets] + } + ) + checks = validator.validate + + refute validator.acceptable?(checks) + assert_equal "skip", checks.find { _1[:id] == "position_surplus" }.fetch(:status) + assert_equal "fail", checks.find { _1[:id] == "position_single_component" }.fetch(:status) + end + + test "requires detailed revenue and expense leaves to reconcile to headline totals" do + line_items = [ + line_item("revenue", "Taxes", "Property taxes", "50,000", 50_000_000, 0), + line_item("revenue", "Transfers", "Government transfers (Note 1)", "30,000", 30_000_000, 1), + line_item("expense", "Services", "Operations", "70,000", 70_000_000, 0) + ] + text = page_texts.merge( + 3 => "Property taxes 50,000 Government transfers 2024 30,000 2025 (Note 1) Operations 70,000" + ) + validator = Validator.new(facts: clean_facts, line_items:, fiscal_year: 2025, page_texts: text) + + assert validator.acceptable? + line_sum_statuses = validator.validate.filter_map do |check| + check[:status] if check[:id].start_with?("line_sum:") + end + assert_equal %w[pass pass], line_sum_statuses + + line_items.first[:value] = BigDecimal("40000000") + refute Validator.new(facts: clean_facts, line_items:, fiscal_year: 2025, page_texts: text).acceptable? + end + + test "reconciles detailed revenue including separately presented adjustments" do + facts = clean_facts.map(&:dup) + annual = facts.find { |fact| fact[:concept] == "annual_surplus" } + annual[:raw_text] = "20,000" + annual[:value] = BigDecimal("20000000") + line_items = [ + line_item("revenue", "Revenue", "Operating revenue", "80,000", 80_000_000, 0), + line_item("revenue", "Other", "Capital contributions", "10,000", 10_000_000, 1), + line_item("expense", "Services", "Operations", "70,000", 70_000_000, 0) + ] + text = page_texts.merge( + 2 => page_texts.fetch(2).sub("10,000", "20,000") + " Other contributions Adjustment", + 3 => "Operating revenue 80,000 Capital contributions 10,000 Operations 70,000" + ) + validator = Validator.new( + facts:, line_items:, fiscal_year: 2025, page_texts: text, + flags: { operations_adjustment_present: true, rollforward_adjustment_present: true } + ) + + assert_equal "pass", validator.validate.find { |check| check[:id] == "line_sum:revenue" }.fetch(:status) + end + + test "prefers the printed revenue total when it already includes the adjustment" do + facts = clean_facts.map(&:dup) + annual = facts.find { |fact| fact[:concept] == "annual_surplus" } + annual[:raw_text] = "15,000" + annual[:value] = BigDecimal("15000000") + line_items = [ + line_item("revenue", "Revenue", "Revenue including contributions", "80,000", 80_000_000, 0), + line_item("expense", "Services", "Operations", "70,000", 70_000_000, 0) + ] + text = page_texts.merge( + 2 => page_texts.fetch(2).sub("10,000", "15,000") + " Capital contributions Adjustment", + 3 => "Revenue including contributions 80,000 Operations 70,000" + ) + validator = Validator.new( + facts:, line_items:, fiscal_year: 2025, page_texts: text, + flags: { operations_adjustment_present: true, rollforward_adjustment_present: true } + ) + + check = validator.validate.find { _1[:id] == "line_sum:revenue" } + assert_equal "pass", check.fetch(:status) + assert_includes check.fetch(:detail), "headline=80000000.0" + end + private def clean_facts @@ -80,6 +209,14 @@ def fact(concept, raw_label, raw_text, value, statement, source_page) } end + def line_item(flow, category, label, raw_text, value, position) + { + flow:, category:, label:, raw_text:, value: BigDecimal(value.to_s), scale: 1_000, + source_page: 3, column_year: "Actual 2025", position:, + extraction_confidence: BigDecimal("0.95") + } + end + def page_texts { 1 => "Financial assets 100,000 Liabilities 60,000 Net financial assets 40,000 Non-financial assets 160,000 Accumulated surplus 200,000", diff --git a/test/models/warehouse/financial_statement_extraction_test.rb b/test/models/warehouse/financial_statement_extraction_test.rb index 46c77d5b..8ffe5366 100644 --- a/test/models/warehouse/financial_statement_extraction_test.rb +++ b/test/models/warehouse/financial_statement_extraction_test.rb @@ -1,6 +1,33 @@ require "test_helper" class Warehouse::FinancialStatementExtractionTest < ActiveSupport::TestCase + test "normalizes saved verification checks for API and audit artifacts" do + checks = Warehouse::FinancialStatementExtraction.verification_checks([ + { "id" => "source_identity", "status" => "pass", "detail" => "source matched", "ignored" => true }, + { id: "line_sum:revenue", status: "fail", detail: "totals differ" } + ]) + + assert_equal [ + { id: "source_identity", status: "pass", detail: "source matched" }, + { id: "line_sum:revenue", status: "fail", detail: "totals differ" } + ], checks + end + + class UnsupportedTestPipeline + PARSER_VERSION = "unsupported-test-v1" + Unsupported = Class.new(StandardError) + + def self.applicable?(**) = true + def initialize(**) = nil + def run = raise(Unsupported, "layout is not supported") + end + + class UnsupportedTestProcessor < Warehouse::FinancialStatementExtraction::QuebecFormProcessor + private + + def pipeline_class = UnsupportedTestPipeline + end + setup do @release = Warehouse::InstitutionRelease.create!( version: "2026-08-27", effective_on: Date.new(2026, 8, 27), schema_version: "1.0", @@ -36,7 +63,7 @@ class Warehouse::FinancialStatementExtractionTest < ActiveSupport::TestCase asset_sha256: @asset.content_sha256, fiscal_year_end: Date.new(2025, 12, 31), extractor_version: "test-v1", - status: "extracted" + status: "extracted", check_results: verification_checks ) extraction.approve!(reviewer: "reviewer") @@ -60,4 +87,169 @@ class Warehouse::FinancialStatementExtractionTest < ActiveSupport::TestCase assert_includes extraction.errors[:asset_sha256], "must identify an archived asset on the release document" end + + test "fiscal year must match the canonical source document year" do + extraction = Warehouse::FinancialStatementExtraction.new( + institution_release: @release, institution_canonical_id: @institution.canonical_id, + document_canonical_id: @document.canonical_id, asset_sha256: @asset.content_sha256, + fiscal_year_end: Date.new(2024, 12, 31), extractor_version: "test-v1", status: "extracted" + ) + + refute extraction.valid? + assert_includes extraction.errors[:fiscal_year_end], + "year must match the source document canonical ID" + end + + test "archive processor is idempotent for an existing reviewed detail extraction" do + extraction = Warehouse::FinancialStatementExtraction.create!( + institution_release: @release, institution_canonical_id: @institution.canonical_id, + document_canonical_id: @document.canonical_id, asset_sha256: @asset.content_sha256, + fiscal_year_end: Date.new(2025, 12, 31), + extractor_version: Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION, + status: "extracted", check_results: verification_checks + ) + extraction.approve!(reviewer: "reviewer") + candidate = Warehouse::FinancialStatementExtraction::CandidateSet::Candidate.new( + document_id: @document.id, institution_canonical_id: @institution.canonical_id, + institution_name: @institution.name_en, document_canonical_id: @document.canonical_id, + asset_sha256: @asset.content_sha256, fiscal_year_end: Date.new(2025, 12, 31), + pdf_path: Pathname("missing-but-unneeded.pdf"), population: nil + ) + + result = Warehouse::FinancialStatementExtraction::Processor.new(release: @release).call(candidate) + + assert_equal "skipped", result.status + assert_equal extraction.id, result.extraction_id + end + + test "review rerun saves details without promoting a headline that still needs review" do + headline = extraction_for( + Warehouse::FinancialStatementExtraction::Pipeline::EXTRACTOR_VERSION, + status: "needs_review" + ) + detailed = extraction_for( + Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION, + status: "needs_review" + ) + pdf_path = Pathname(Dir.mktmpdir).join("statement.pdf") + pdf_path.write("source") + candidate = candidate(pdf_path:) + processor = Warehouse::FinancialStatementExtraction::Processor.new( + release: @release, rerun: "review" + ) + headline_called = false + detailed_called = false + + processor.stub(:run_headline, ->(_candidate, extraction) { headline_called = true; extraction }) do + processor.stub(:run_detailed, ->(*) { detailed_called = true; detailed }) do + result = processor.call(candidate) + + assert_equal "needs_review", result.status + assert_equal "detailed", result.stage + assert_equal detailed.id, result.extraction_id + end + end + assert headline_called + assert detailed_called + assert_equal "needs_review", detailed.reload.status + ensure + FileUtils.remove_entry(pdf_path.dirname) if pdf_path&.dirname&.directory? + end + + test "deterministic unsupported outcomes persist a failed check ledger" do + pdf_path = Pathname(Dir.mktmpdir).join("statement.pdf") + pdf_path.write("source") + + result = UnsupportedTestProcessor.new(release: @release).call(candidate(pdf_path:)) + extraction = @release.financial_statement_extractions.find_by!( + asset_sha256: @asset.content_sha256, + extractor_version: Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION + ) + + assert_equal "unsupported", result.status + assert_equal "failed", extraction.status + assert_equal "deterministic_parser", extraction.check_results.sole.fetch("id") + assert_equal "fail", extraction.check_results.sole.fetch("status") + assert_equal "unsupported", extraction.llm_response_snapshot.fetch("outcome") + ensure + FileUtils.remove_entry(pdf_path.dirname) if pdf_path&.dirname&.directory? + end + + test "approval requires saved verification results" do + extraction = Warehouse::FinancialStatementExtraction.new( + institution_release: @release, institution_canonical_id: @institution.canonical_id, + document_canonical_id: @document.canonical_id, asset_sha256: @asset.content_sha256, + fiscal_year_end: Date.new(2025, 12, 31), extractor_version: "test-v1", status: "extracted" + ) + + error = assert_raises(ArgumentError) { extraction.approve!(reviewer: "reviewer") } + + assert_includes error.message, "verification check results" + assert_equal "extracted", extraction.status + end + + test "all completed statuses require saved verification results" do + extraction = Warehouse::FinancialStatementExtraction.new( + institution_release: @release, institution_canonical_id: @institution.canonical_id, + document_canonical_id: @document.canonical_id, asset_sha256: @asset.content_sha256, + fiscal_year_end: Date.new(2025, 12, 31), extractor_version: "test-v1", status: "failed" + ) + + refute extraction.valid? + assert_includes extraction.errors[:check_results], "must be saved for a completed extraction" + end + + test "extraction exceptions persist a failed check ledger" do + extraction = extraction_for( + Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION, + status: "extracting" + ) + error = RubyLLM::BadRequestError.new("invalid request") + + extraction.extractor.send(:record_failure, error, stage: "detailed_extraction") + + extraction.reload + assert_equal "failed", extraction.status + assert_equal "detailed_extraction", extraction.check_results.sole.fetch("id") + assert_equal "fail", extraction.check_results.sole.fetch("status") + assert_includes extraction.check_results.sole.fetch("detail"), "invalid request" + end + + test "direct approval requires review provenance" do + extraction = Warehouse::FinancialStatementExtraction.create!( + institution_release: @release, institution_canonical_id: @institution.canonical_id, + document_canonical_id: @document.canonical_id, asset_sha256: @asset.content_sha256, + fiscal_year_end: Date.new(2025, 12, 31), extractor_version: "test-v1", + status: "extracted", check_results: verification_checks + ) + + error = assert_raises(ActiveRecord::RecordInvalid) { extraction.update!(status: "approved") } + + assert_includes error.record.errors[:reviewed_at], "and reviewer must be saved before approval" + assert_equal "extracted", extraction.reload.status + end + + private + + def extraction_for(version, status:) + Warehouse::FinancialStatementExtraction.create!( + institution_release: @release, institution_canonical_id: @institution.canonical_id, + document_canonical_id: @document.canonical_id, asset_sha256: @asset.content_sha256, + fiscal_year_end: Date.new(2025, 12, 31), extractor_version: version, status:, + check_results: status.in?(%w[extracted needs_review approved rejected failed]) ? verification_checks : [] + ) + end + + def candidate(pdf_path:) + Warehouse::FinancialStatementExtraction::CandidateSet::Candidate.new( + document_id: @document.id, institution_canonical_id: @institution.canonical_id, + institution_name: @institution.name_en, document_canonical_id: @document.canonical_id, + asset_sha256: @asset.content_sha256, fiscal_year_end: Date.new(2025, 12, 31), + pdf_path:, population: nil + ) + end + + def verification_checks + [ { id: "source_identity", status: "pass", detail: "source hash matches" } ] + end end diff --git a/test/scripts/audit_municipal_financial_extraction_coverage_test.rb b/test/scripts/audit_municipal_financial_extraction_coverage_test.rb new file mode 100644 index 00000000..674223a9 --- /dev/null +++ b/test/scripts/audit_municipal_financial_extraction_coverage_test.rb @@ -0,0 +1,156 @@ +require "test_helper" + +class AuditMunicipalFinancialExtractionCoverageTest < ActiveSupport::TestCase + setup do + @release = Warehouse::InstitutionRelease.create!( + version: "2026-08-30", effective_on: Date.new(2026, 8, 30), + schema_version: "1.0", published_at: Time.utc(2026, 8, 30), geography_vintage: 2021, + attribution: "Test" + ) + @source = Warehouse::InstitutionSource.create!( + institution_release: @release, canonical_id: "ca/sources/coverage-audit-test", + publisher_name: "Test", title_en: "Test", url: "https://example.test/source", + retrieved_at: @release.published_at, languages: [ "en" ] + ) + end + + test "classifies shared assets and headline gates without duplicating facts" do + predicted_owner = create_document("ca/nb/predicted-owner", 2025, "a" * 64) + persisted_owner = create_document("ca/nb/persisted-owner", 2025, "a" * 64) + approved = create_detailed_extraction(persisted_owner, "a" * 64) + + unattempted_owner = create_document("ca/nb/unattempted-owner", 2024, "b" * 64) + unattempted_alias = create_document("ca/nb/unattempted-alias", 2024, "b" * 64) + headline_document = create_document("ca/nb/headline-gate", 2023, "c" * 64) + create_failed_headline_extraction(headline_document, "c" * 64) + + payload = Warehouse::FinancialStatementExtraction::CoverageAudit.new( + release: @release, provinces: [ "nb" ] + ).payload + records = payload.fetch(:records).index_by { _1.fetch(:document_canonical_id) } + + persisted_alias = records.fetch(predicted_owner.canonical_id) + assert_equal "shared_asset", persisted_alias.fetch(:status) + assert_equal "shared_asset", persisted_alias.fetch(:extraction_stage) + assert_equal approved.id, persisted_alias.fetch(:extraction_id) + assert_equal persisted_owner.canonical_id, + persisted_alias.fetch(:shared_with_document_canonical_id) + assert_equal "ca/nb/persisted-owner", + persisted_alias.fetch(:shared_with_institution_canonical_id) + assert_equal "approved", persisted_alias.fetch(:shared_extraction_status) + assert_equal 1, persisted_alias.dig(:verification, :total) + assert_equal 0, persisted_alias.fetch(:fact_count) + assert_equal 0, persisted_alias.fetch(:line_item_count) + + owner_record = records.fetch(persisted_owner.canonical_id) + assert_equal "approved", owner_record.fetch(:status) + assert_equal 1, owner_record.fetch(:fact_count) + assert_equal 1, owner_record.fetch(:line_item_count) + + predicted_alias = records.fetch(unattempted_alias.canonical_id) + assert_equal "shared_asset", predicted_alias.fetch(:status) + assert_nil predicted_alias.fetch(:extraction_id) + assert_equal unattempted_owner.canonical_id, + predicted_alias.fetch(:shared_with_document_canonical_id) + assert_equal 0, predicted_alias.dig(:verification, :total) + assert_equal "unattempted", records.fetch(unattempted_owner.canonical_id).fetch(:status) + + headline_gate = records.fetch(headline_document.canonical_id) + assert_equal "failed_headline_gate", headline_gate.fetch(:status) + assert_equal "headline_gate", headline_gate.fetch(:extraction_stage) + assert_equal 1, headline_gate.dig(:verification, :total) + assert_equal "fail", headline_gate.dig(:verification, :checks, 0, :status) + + totals = payload.fetch(:totals) + assert_equal 5, totals.fetch(:preferred_asset_count) + assert_equal({ "approved" => 1, "shared_asset" => 2, "unattempted" => 1, + "failed_headline_gate" => 1 }, totals.fetch(:status_counts)) + assert_equal 4, totals.dig(:reconciliation, :classified_asset_count) + assert_equal 1, totals.dig(:reconciliation, :unclassified_asset_count) + assert_equal totals.fetch(:preferred_asset_count), + totals.dig(:reconciliation, :classified_asset_count) + + totals.dig(:reconciliation, :unclassified_asset_count) + assert_equal 0, totals.fetch(:failed_headline_gate_without_checks) + assert_equal 0, totals.fetch(:shared_asset_with_terminal_extraction_without_checks) + end + + test "accepts source and visual deterministic reviewer provenance" do + source_reviewed = create_document("ca/nb/source-reviewed", 2025, "d" * 64) + create_detailed_extraction(source_reviewed, "d" * 64) + visual_reviewed = create_document("ca/nb/visual-reviewed", 2024, "e" * 64) + create_detailed_extraction( + visual_reviewed, "e" * 64, + reviewer: Warehouse::FinancialStatementExtraction::Reviewer::VISUAL_REVIEWER + ) + legacy_reviewed = create_document("ca/nb/legacy-reviewed", 2023, "f" * 64) + create_detailed_extraction(legacy_reviewed, "f" * 64, reviewer: "legacy-local-reviewer") + + totals = Warehouse::FinancialStatementExtraction::CoverageAudit.new( + release: @release, provinces: [ "nb" ] + ).payload.fetch(:totals) + + assert_equal 1, totals.fetch(:approved_without_deterministic_reviewer) + end + + private + + def create_document(canonical_id, year, sha) + institution = Warehouse::Institution.create!( + institution_release: @release, institution_source: @source, canonical_id:, + name_en: canonical_id.split("/").last.titleize, institution_type: "government", + government_level: "municipal", status: "active" + ) + document = Warehouse::InstitutionDocument.create!( + institution_release: @release, institution:, institution_source: @source, + canonical_id: "#{canonical_id}/documents/financial-statements/#{year}/general", + document_type: "financial-statements", document_variant: "general", + fiscal_period_end: Date.new(year, 12, 31) + ) + Warehouse::InstitutionDocumentAsset.create!( + institution_release: @release, institution_document: document, content_sha256: sha, + asset_role: "final", preferred: true, download_url: "https://example.test/#{sha}.pdf", + retrieved_at: @release.published_at, archive_path: "sha256/#{sha.first(2)}/#{sha}.pdf", + mime_type: "application/pdf", byte_size: 100, rights_status: "metadata_only" + ) + document + end + + def create_detailed_extraction( + document, sha, reviewer: Warehouse::FinancialStatementExtraction::Reviewer::REVIEWER + ) + extraction = Warehouse::FinancialStatementExtraction.create!( + institution_release: @release, institution_canonical_id: document.institution.canonical_id, + document_canonical_id: document.canonical_id, asset_sha256: sha, + fiscal_year_end: document.fiscal_period_end, + extractor_version: Warehouse::FinancialStatementExtraction::DetailedPipeline::EXTRACTOR_VERSION, + status: "extracted", check_results: [ saved_check("pass") ] + ) + extraction.financial_statement_facts.create!( + concept: "total_revenue", value: 100, raw_text: "100", raw_label: "Total revenue", + scale: 1, statement: "operations", source_page: 1, column_year: "2025", + extraction_confidence: 1 + ) + extraction.financial_statement_line_items.create!( + flow: "revenue", category: "Taxes", label: "Property tax", value: 100, + raw_text: "100", scale: 1, source_page: 1, column_year: "2025", position: 0, + extraction_confidence: 1 + ) + extraction.approve!(reviewer:) + extraction + end + + def create_failed_headline_extraction(document, sha) + Warehouse::FinancialStatementExtraction.create!( + institution_release: @release, institution_canonical_id: document.institution.canonical_id, + document_canonical_id: document.canonical_id, asset_sha256: sha, + fiscal_year_end: document.fiscal_period_end, + extractor_version: Warehouse::FinancialStatementExtraction::Pipeline::EXTRACTOR_VERSION, + status: "failed", error_message: "statement pages not found", + check_results: [ saved_check("fail") ] + ) + end + + def saved_check(status) + { id: "source_identity", status:, detail: "saved result" } + end +end diff --git a/test/scripts/process_municipal_financial_statements_test.rb b/test/scripts/process_municipal_financial_statements_test.rb new file mode 100644 index 00000000..8c23cc1a --- /dev/null +++ b/test/scripts/process_municipal_financial_statements_test.rb @@ -0,0 +1,37 @@ +require "test_helper" +require "open3" + +class ProcessMunicipalFinancialStatementsTest < ActiveSupport::TestCase + SCRIPT = Rails.root.join("script/process_municipal_financial_statements.rb").to_s + + test "rejects a failed extractor without failed-only mode" do + _stdout, stderr, status = Open3.capture3( + RbConfig.ruby, SCRIPT, "--province", "on", "--failed-extractor", "headline", + chdir: Rails.root.to_s + ) + + refute status.success? + assert_includes stderr, "--failed-extractor requires --failed-only" + end + + test "rejects an unknown failed extractor" do + _stdout, stderr, status = Open3.capture3( + RbConfig.ruby, SCRIPT, "--province", "on", "--failed-extractor", "unknown", + chdir: Rails.root.to_s + ) + + refute status.success? + assert_includes stderr, "invalid argument: --failed-extractor unknown" + end + + test "rejects parser targeting for headline failures" do + _stdout, stderr, status = Open3.capture3( + RbConfig.ruby, SCRIPT, "--province", "on", "--rerun", "failed", "--failed-only", + "--failed-extractor", "headline", "--failed-parser", "parser-v1", + chdir: Rails.root.to_s + ) + + refute status.success? + assert_includes stderr, "--failed-parser only supports the detailed failed extractor" + end +end diff --git a/test/scripts/sanitize_municipal_report_batch_test.rb b/test/scripts/sanitize_municipal_report_batch_test.rb index c5b25023..3f6ad579 100644 --- a/test/scripts/sanitize_municipal_report_batch_test.rb +++ b/test/scripts/sanitize_municipal_report_batch_test.rb @@ -55,4 +55,41 @@ def test_content_year_uses_the_earliest_primary_period_evidence assert_equal 2019, @sanitizer.send(:corrected_year, report, text) end + + def test_rejects_greater_sudbury_utilities_corporate_statements + [ + "2009 - Consolidated Financial Statements Of The Greater Sudbury Utilities Inc.", + "2010 - Consolidated Financial Statements Of The Greater Sudbury Utilities Inc." + ].each do |title| + report = financial_statement_report(title) + + assert_equal "subsidiary or trust financial statements, not the reporting institution's statements", + @sanitizer.send(:rejection_reason, report, "") + end + end + + def test_retains_city_statements_that_discuss_utility_operations + report = financial_statement_report("Consolidated Financial Statements of the City of Greater Sudbury") + text = "The municipality reports utility revenue and utility expenses in its consolidated operations." + + assert_nil @sanitizer.send(:rejection_reason, report, text) + end + + def test_retains_municipal_statements_with_bare_plural_utilities_fund_title + report = financial_statement_report("Consolidated Financial Statements — General and Utilities Funds") + + assert_nil @sanitizer.send(:rejection_reason, report, "") + end + + private + + def financial_statement_report(title) + { + "year" => 2024, + "document_type" => "financial-statements", + "download_url" => "https://example.ca/statements.pdf", + "source_page_url" => "https://example.ca/finance", + "title" => title + } + end end From b823bec2069dda5527179a69d30b2c186c4f7763 Mon Sep 17 00:00:00 2001 From: xrendan Date: Mon, 31 Aug 2026 08:55:44 -0600 Subject: [PATCH 2/3] Harden municipal financial deployment process --- ...ire_approved_financial_statement_checks.rb | 4 +- ...ire_approved_financial_statement_review.rb | 4 +- ...re_completed_financial_statement_checks.rb | 4 +- ...nicipal_financial_statements_deployment.md | 53 ++++++++++++------- ..._municipal_deployment_review_2026-08-31.md | 32 +++++++++++ ...reflight_municipal_financial_migrations.rb | 28 ++++++++++ 6 files changed, 104 insertions(+), 21 deletions(-) create mode 100644 docs/reviews/fable_municipal_deployment_review_2026-08-31.md create mode 100644 script/preflight_municipal_financial_migrations.rb diff --git a/db/migrate/20260829000005_require_approved_financial_statement_checks.rb b/db/migrate/20260829000005_require_approved_financial_statement_checks.rb index a3f202fb..0f3f6dff 100644 --- a/db/migrate/20260829000005_require_approved_financial_statement_checks.rb +++ b/db/migrate/20260829000005_require_approved_financial_statement_checks.rb @@ -3,6 +3,8 @@ def change add_check_constraint :financial_statement_extractions, "status <> 'approved' OR (jsonb_typeof(check_results) = 'array' AND jsonb_array_length(check_results) > 0)", name: "financial_statement_extractions_approved_checks", - schema: "warehouse" + schema: "warehouse", validate: false + validate_check_constraint :financial_statement_extractions, + name: "financial_statement_extractions_approved_checks", schema: "warehouse" end end diff --git a/db/migrate/20260829000006_require_approved_financial_statement_review.rb b/db/migrate/20260829000006_require_approved_financial_statement_review.rb index c2e9e15e..d6542b5c 100644 --- a/db/migrate/20260829000006_require_approved_financial_statement_review.rb +++ b/db/migrate/20260829000006_require_approved_financial_statement_review.rb @@ -3,6 +3,8 @@ def change add_check_constraint :financial_statement_extractions, "status <> 'approved' OR (reviewed_at IS NOT NULL AND reviewed_by IS NOT NULL)", name: "financial_statement_extractions_approved_review", - schema: "warehouse" + schema: "warehouse", validate: false + validate_check_constraint :financial_statement_extractions, + name: "financial_statement_extractions_approved_review", schema: "warehouse" end end diff --git a/db/migrate/20260829000007_require_completed_financial_statement_checks.rb b/db/migrate/20260829000007_require_completed_financial_statement_checks.rb index a552291d..5d3bdf98 100644 --- a/db/migrate/20260829000007_require_completed_financial_statement_checks.rb +++ b/db/migrate/20260829000007_require_completed_financial_statement_checks.rb @@ -4,6 +4,8 @@ def change "status NOT IN ('extracted', 'needs_review', 'approved', 'rejected', 'failed') " \ "OR (jsonb_typeof(check_results) = 'array' AND jsonb_array_length(check_results) > 0)", name: "financial_statement_extractions_completed_checks", - schema: "warehouse" + schema: "warehouse", validate: false + validate_check_constraint :financial_statement_extractions, + name: "financial_statement_extractions_completed_checks", schema: "warehouse" end end diff --git a/docs/plans/municipal_financial_statements_deployment.md b/docs/plans/municipal_financial_statements_deployment.md index cffd2a8f..05797b42 100644 --- a/docs/plans/municipal_financial_statements_deployment.md +++ b/docs/plans/municipal_financial_statements_deployment.md @@ -28,11 +28,15 @@ Use release `2026-08-27` for the current national run. Before exporting it: 3. Preserve the final coverage, numeric, scale, lineage, issuer, and test-result artifacts alongside the release. Failures and unavailable documents remain explicit; they are not silently dropped. -4. Run the York Factory test suite and the CanadaSpends Vitest suite/build from +4. Upload the finalizer, coverage, numeric, scale, lineage, issuer, and test logs + to the immutable release's R2 `audit/` prefix. Files left only on + `/Volumes/floppy` do not satisfy this gate. +5. Run the York Factory test suite and the CanadaSpends Vitest suite/build from the exact commits being deployed. -5. Export the immutable ontology release only after its `published_at` includes - the final review timestamps. Never edit an already-published release in place; - if the release has already been published, create a new dated release. +6. Freeze the financial-data bundle at an explicit review cutoff. The ontology + release's `published_at` predates these extractions, so its current exporter + would emit none of the newly reviewed rows. Never change or recut an existing + bundle; corrections use a new dated bundle. ## Data promotion contract @@ -45,7 +49,7 @@ bin/rails institution_ontology:export[2026-08-27,tmp/public-institutions-2026-08 The exported directory must contain the ontology, documents, document-asset metadata, approved extractions, facts, detailed revenue/expense line items, -census context, manifest, SQL loader, and `SHA256SUMS`. Upload it under a + census context, manifest, SQL loader, and `checksums.sha256`. Upload it under a versioned R2 key such as `municipal-financial-statements/releases/2026-08-27/`; never overwrite that key. Archived source binaries remain content-addressed by SHA-256 in the archival R2 @@ -66,24 +70,34 @@ existing Warehouse release. That importer must: - stage and validate all rows, then make them visible atomically; - provide a dry-run that performs every validation without writing. -Until that importer is implemented and tested, the supported fallback is to run -the deterministic extraction/review pipeline against the imported immutable -source release in production. A whole-database dump/restore is not a supported -promotion path. +Until that importer and the immutable R2 uploader are implemented and tested, +production data promotion is blocked. Re-running the extraction pipeline in +production would re-derive rather than promote the reviewed result and is not a +supported fallback. A whole-database dump/restore is also not supported. ## York Factory deployment -After PR #111 and the stacked York Factory PR merge: +Production currently predates the institution-ontology tables. Deploy in two +York steps: merge and deploy PR #111 first, initialize the checksummed source +release through its recipe, then run the read-only migration preflight: + +```sh +bin/rails runner script/preflight_municipal_financial_migrations.rb +``` + +Only after that succeeds should the stacked York Factory PR merge and deploy: ```sh bin/kamal deploy bin/kamal app exec --reuse 'bin/rails db:migrate:status' ``` -The seven municipal migrations are additive except for replacing extraction and -line-item unique indexes and adding check constraints. Before deploy, verify the -production table has no rows that violate the new constraints. Keep the API -unreferenced by CanadaSpends until migrations and data promotion succeed. +The web entrypoint runs `db:prepare`, so migrations execute as the new container +starts. The seven municipal migrations are additive except for replacing +extraction and line-item unique indexes and adding validated check constraints. +The constraints are added `NOT VALID` and then validated to avoid taking the +strongest table lock for the scan. Keep the API unreferenced by CanadaSpends +until migrations and data promotion succeed. Promote data only through the verified importer described above (first with `DRY_RUN=1`), then verify at minimum: @@ -100,7 +114,9 @@ Sankey data where detailed line items passed validation. ## CanadaSpends deployment -Set the production server-side environment variable: +Set the production server-side environment variable in the Cloudflare Workers +dashboard (and explicitly configure preview deployments rather than allowing +them to inherit an accidental origin): ```text YORK_FACTORY_API_URL=https://yorkfactory.buildcanada.com/api/v1 @@ -119,6 +135,7 @@ origin and its URL must not be committed or configured in production. - Roll CanadaSpends back first; York's new API can remain unused. - Roll York code back only while leaving additive tables/columns in place. Do not reverse schema migrations during an incident. -- If a data payload is wrong, disable the frontend route or mark the affected - release unpublished, retain the import audit, and promote a corrected new - release. Do not mutate the immutable R2 key or erase failed test evidence. +- If a data payload is wrong, disable the frontend route, retain the import + audit, and promote a corrected, superseding dated release. The current release + model has no unpublished state. Do not mutate the immutable R2 key or erase + failed test evidence. diff --git a/docs/reviews/fable_municipal_deployment_review_2026-08-31.md b/docs/reviews/fable_municipal_deployment_review_2026-08-31.md new file mode 100644 index 00000000..3ca63952 --- /dev/null +++ b/docs/reviews/fable_municipal_deployment_review_2026-08-31.md @@ -0,0 +1,32 @@ +# Fable review — municipal financial deployment + +Claude Fable reviewed the two-repository release process on 2026-08-31 in the +`municipal-pr-deploy-fable-review` tmux session. The review covered both git +diffs, stacked PR ancestry, York's migrations and Kamal entrypoint, the immutable +release exporter, R2 assumptions, CanadaSpends runtime behavior, and rollback. + +The review agreed with the parent-then-stack and York-before-CanadaSpends order, +but rejected the original data-promotion section as non-executable. In +particular: + +- the existing ontology exporter omits detailed line items and census profiles; +- its SQL loader populates `public_institutions`, not the Rails `warehouse` + tables queried by the API; +- no verified Warehouse importer or immutable R2 release uploader exists; +- the production deployment predates the ontology tables, so the source release + must be initialized before detailed data can be imported; +- final evidence currently lives on `/Volumes/floppy` and must be uploaded with + the immutable release before production promotion; +- the Rails container runs `db:prepare` on web boot, so constraint compatibility + must be checked before the stacked York deploy; +- a bad release cannot currently be marked unpublished; correction requires + disabling the frontend route and importing a superseding dated release. + +Fable also identified shared CanadaSpends risks. The follow-up fixes preserve +declared Sankey denominators when present, promote compact currency values across +rounding boundaries, derive latest/earliest years without trusting API order, and +restore yearless municipal metadata/fallback behavior. + +The revised deployment plan treats the verified importer, R2 uploader, frozen +audit artifact set, and migration preflight as hard gates. It explicitly rejects +whole-database copying and production re-extraction as promotion mechanisms. diff --git a/script/preflight_municipal_financial_migrations.rb b/script/preflight_municipal_financial_migrations.rb new file mode 100644 index 00000000..4013f614 --- /dev/null +++ b/script/preflight_municipal_financial_migrations.rb @@ -0,0 +1,28 @@ +# Run after the public-institution ontology deploy and before deploying the +# stacked municipal financial-statements PR. +connection = Warehouse::FinancialStatementExtraction.connection +table = "warehouse.financial_statement_extractions" +abort "#{table} does not exist; deploy the parent ontology PR first" unless connection.data_source_exists?(table) + +terminal = %w[extracted needs_review approved rejected failed] +scope = Warehouse::FinancialStatementExtraction.where(status: terminal) +missing_checks = scope.where(<<~SQL.squish).count + check_results IS NULL + OR jsonb_typeof(check_results) <> 'array' + OR jsonb_array_length(check_results) = 0 +SQL +approved_without_review = Warehouse::FinancialStatementExtraction.where(status: "approved") + .where("reviewed_at IS NULL OR reviewed_by IS NULL").count +unpaired_review = Warehouse::FinancialStatementExtraction + .where("(reviewed_at IS NULL) <> (reviewed_by IS NULL)").count + +result = { + table:, + terminal_rows: scope.count, + terminal_without_checks: missing_checks, + approved_without_review:, + unpaired_review:, + ready: missing_checks.zero? && approved_without_review.zero? && unpaired_review.zero? +} +puts result.to_json +abort "municipal financial migration preflight failed" unless result.fetch(:ready) From 3bdb338e999bea80201742eb37415e22bd6c9ba8 Mon Sep 17 00:00:00 2001 From: xrendan Date: Mon, 31 Aug 2026 08:59:14 -0600 Subject: [PATCH 3/3] Remove raw SQL from municipal pagination --- ...nicipal_financial_statements_controller.rb | 43 ++++++++----------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/app/controllers/api/v1/warehouse/municipal_financial_statements_controller.rb b/app/controllers/api/v1/warehouse/municipal_financial_statements_controller.rb index b1159f26..fe970d14 100644 --- a/app/controllers/api/v1/warehouse/municipal_financial_statements_controller.rb +++ b/app/controllers/api/v1/warehouse/municipal_financial_statements_controller.rb @@ -111,31 +111,26 @@ def latest_approved_extractions_scope end def paginated_municipality_ids(scope, page:, per_page:) - connection = ::Warehouse::FinancialStatementExtraction.connection - latest_sql = scope.to_sql - statement_count = connection.select_value(<<~SQL.squish).to_i - SELECT COUNT(*) FROM (#{latest_sql}) latest_financial_statements - SQL - municipality_count = connection.select_value(<<~SQL.squish).to_i - SELECT COUNT(DISTINCT institution_canonical_id) - FROM (#{latest_sql}) latest_financial_statements - SQL + rows = scope.unscope(:select, :order).pluck( + :institution_release_id, :institution_canonical_id, :fiscal_year_end + ) + statement_count = rows.map { |_, canonical_id, period| [ canonical_id, period ] }.uniq.length + canonical_ids = rows.map { |_, canonical_id, _| canonical_id }.uniq + release_ids = rows.map(&:first).uniq + names = ::Warehouse::Institution.joins(:institution_release) + .where(institution_release_id: release_ids, canonical_id: canonical_ids) + .pluck(:canonical_id, :name_en, :name_fr, "warehouse.institution_releases.effective_on") + .group_by(&:first) + .transform_values do |versions| + newest = versions.max_by { |_, _, _, effective_on| effective_on } + newest[1].presence || newest[2].presence || newest[0] + end + ordered_ids = canonical_ids.sort_by do |canonical_id| + [ canonical_id.split("/").fetch(1, ""), names.fetch(canonical_id, canonical_id).downcase, canonical_id ] + end + municipality_count = ordered_ids.length offset = (page - 1) * per_page - canonical_ids = connection.select_values(<<~SQL.squish) - WITH latest_financial_statements AS (#{latest_sql}) - SELECT latest_financial_statements.institution_canonical_id - FROM latest_financial_statements - INNER JOIN warehouse.institutions index_institutions - ON index_institutions.institution_release_id = latest_financial_statements.institution_release_id - AND index_institutions.canonical_id = latest_financial_statements.institution_canonical_id - GROUP BY latest_financial_statements.institution_canonical_id - ORDER BY - split_part(latest_financial_statements.institution_canonical_id, '/', 2), - LOWER(MAX(COALESCE(index_institutions.name_en, index_institutions.name_fr, ''))), - latest_financial_statements.institution_canonical_id - LIMIT #{per_page} OFFSET #{offset} - SQL - [ canonical_ids, municipality_count, statement_count ] + [ ordered_ids.slice(offset, per_page) || [], municipality_count, statement_count ] end def institutions_by_key(extractions)