diff --git a/dependencies/requirements.test.txt b/dependencies/requirements.test.txt index 22ece82..93e0915 100644 --- a/dependencies/requirements.test.txt +++ b/dependencies/requirements.test.txt @@ -4,3 +4,6 @@ ipython==8.23.0 moto[server]==5.1.1 pytest==8.2.0 pytest-asyncio==0.24.0 +pytest-timeout==2.4.0 +pyyaml==6.0.2 +shandy-sqlfmt[jinjafmt]==0.27.0 diff --git a/dependencies/requirements.txt b/dependencies/requirements.txt index eaa438e..6d8d64b 100644 --- a/dependencies/requirements.txt +++ b/dependencies/requirements.txt @@ -1,5 +1,5 @@ boto3==1.37.7 -duckdb==1.3.1 +duckdb==1.3.2 jinja2==3.1.5 pandas==2.2.3 pydantic==2.10.6 diff --git a/ostrich_egg/config.py b/ostrich_egg/config.py index a39262e..da9c24b 100644 --- a/ostrich_egg/config.py +++ b/ostrich_egg/config.py @@ -4,7 +4,14 @@ from typing import Any, Union, Annotated, List, Sequence, Optional, Literal -from pydantic import BaseModel, Field, AliasChoices, TypeAdapter +from pydantic import ( + BaseModel, + Field, + AliasChoices, + TypeAdapter, + AfterValidator, + field_serializer, +) from enum import StrEnum from ostrich_egg.utils import identifier, get_logger, DEFAULT_MASKING_VALUE @@ -36,6 +43,23 @@ class ReplaceWithRedactedParameters(BaseModel): ), ] ] = None + non_summable_dimensions: Optional[ + Annotated[ + Union[List[str], None], + Field( + description="List of dimensions that are part of the dataset but will not ever actually be aggregated. For example, if you have unrelated indicators in a column or years when you won't ever sum the total number of incidences across time (preventing revelation through subtraction). Use with caution." + ), + ] + ] = None + first_order_only: Optional[ + Annotated[ + bool, + Field( + description="Whether to only redact cells to prevent latent revelation along a single axis of dimensions. Enabling reduces the total number of cells suppressed but creates a risk of transitive revelation across dimensions.", + default=False, + ), + ] + ] = False class ReplaceWithRedacted(Strategy): @@ -58,6 +82,15 @@ class MarkRedactedParameters(BaseModel): ), ] ] = None + first_order_only: Optional[ + Annotated[ + bool, + Field( + description="Whether to only redact cells to prevent latent revelation along a single axis of dimensions. Enabling reduces the total number of cells suppressed but creates a risk of transitive revelation across dimensions.", + default=False, + ), + ] + ] = False class MarkRedacted(Strategy): @@ -224,6 +257,37 @@ def should_include_in_initial_state(self, initial: bool = False): return initial_conditions_match or counter_condition_does_not_exclude +class DimensionOrder(BaseModel): + dimension: str + direction: Literal["asc", "desc"] = "asc" + + @property + def sql_expression(self) -> str: + """ + For use in templates so you can just | join(', ') the list. + """ + return f"{identifier(self.dimension)} {self.direction}" + + +def validate_dimension_orders( + dimension_orders: List[Union[str, DimensionOrder]] | None, +): + """ + Partially for backwards compatibility but also ease-of-use, the user could just plausibly specify as list of dimensions and we'll assume they meant to order them in ascending order. + """ + + valid_dimension_orders = [] + if not dimension_orders: + return valid_dimension_orders + + for dimension_order in dimension_orders: + if isinstance(dimension_order, str): + valid_dimension_orders.append(DimensionOrder(dimension=dimension_order)) + elif isinstance(dimension_order, DimensionOrder): + valid_dimension_orders.append(dimension_order) + return valid_dimension_orders + + class Dataset(BaseModel): name: Optional[ @@ -268,13 +332,26 @@ class Dataset(BaseModel): ] = None redaction_order_dimensions: Optional[ Annotated[ - Union[List[str], None], + Union[List[DimensionOrder], List[str], None], + AfterValidator(validate_dimension_orders), Field( default_factory=list, - description="The dimensions to order the redaction by. If not specified, the dimensions will be ordered by the order they are specified in the dataset.", + description="The dimensions to order the redaction by. If not specified, the dimensions will be ordered by the order they are specified in the dataset ascending.", ), ] - ] = None + ] = Field( + default_factory=list, + ) + + @field_serializer("redaction_order_dimensions") + def serialize_redaction_order_dimensions( + self, dimension_orders: List[Union[str, DimensionOrder]] | None + ): + return ( + None + if not dimension_orders + else validate_dimension_orders(dimension_orders) + ) class DataSource(BaseModel): diff --git a/ostrich_egg/connectors/base.py b/ostrich_egg/connectors/base.py index cb686e8..c97e758 100644 --- a/ostrich_egg/connectors/base.py +++ b/ostrich_egg/connectors/base.py @@ -3,6 +3,8 @@ import duckdb +from ostrich_egg.utils import should_redact_along_axis + DEFAULT_TABLE_NAME = "dataset" DEFAULT_RESULT_NAME = "result" @@ -28,6 +30,7 @@ class BaseConnector(AbstractBaseClass): def __init__(self, table_name=DEFAULT_TABLE_NAME, **kwargs): self.table_name = table_name + self.init_duckdb() def __exit__(self): self.duckdb_connection.close() @@ -36,10 +39,19 @@ def __exit__(self): def duckdb_connection(self) -> duckdb.DuckDBPyConnection: return duckdb.connect() + @property + def db(self): + return self.duckdb_connection + def init_duckdb(self): + try: + del self.duckdb_connection + except AttributeError: + pass for extension in self.extensions: self.duckdb_connection.install_extension(extension) self.duckdb_connection.load_extension(extension) + self.load_custom_functions() @abstractmethod def load_source_table(self, *args, **kwargs): @@ -48,3 +60,15 @@ def load_source_table(self, *args, **kwargs): If this were postgres, you wouldn't actually need to create a table. """ raise NotImplementedError("Connectors must implement a create table interface") + + def load_custom_functions(self): + should_redact_function_exists_count = ( + self.db.sql( + "from duckdb_functions() where function_name = 'should_redact_along_axis'" + ) + .count("*") + .fetchone()[0] + ) + if should_redact_function_exists_count == 1: + self.db.remove_function("should_redact_along_axis") + self.db.create_function("should_redact_along_axis", should_redact_along_axis) diff --git a/ostrich_egg/connectors/file_system.py b/ostrich_egg/connectors/file_system.py index 9152b0b..a10fd7d 100644 --- a/ostrich_egg/connectors/file_system.py +++ b/ostrich_egg/connectors/file_system.py @@ -15,7 +15,6 @@ def load_source_table( """ table_name = table_name or self.table_name file_path = source_file or self.file_path - self.init_duckdb() self.duckdb_connection.execute( f"create or replace table {table_name} as (select * from '{file_path}')" ) diff --git a/ostrich_egg/connectors/s3.py b/ostrich_egg/connectors/s3.py index c8a9cc0..78098b0 100644 --- a/ostrich_egg/connectors/s3.py +++ b/ostrich_egg/connectors/s3.py @@ -5,7 +5,7 @@ import duckdb from jinja2 import Template -from ostrich_egg.connectors.base import BaseConnector +from ostrich_egg.connectors.base import BaseConnector, DEFAULT_TABLE_NAME # refer to https://duckdb.org/docs/configuration/secrets_manager DEFAULT_S3_SECRET_NAME = "__default_s3" @@ -45,13 +45,14 @@ def __init__( use_ssl: bool = True, url_style: Literal["vhost", "path"] = "vhost", chain: str = None, + table_name: str = DEFAULT_TABLE_NAME, *args, **kwargs, ): """ See https://duckdb.org/docs/extensions/httpfs/s3api """ - super().__init__(*args, **kwargs) + self.table_name = table_name self.bucket = bucket self.key = key self.region = region @@ -63,6 +64,7 @@ def __init__( self.endpoint = endpoint self.use_ssl = use_ssl self.url_style = url_style + self.init_duckdb() def __exit__(self): self.duckdb_connection.close() @@ -151,7 +153,6 @@ def load_source_table( bucket = bucket or self.bucket key = source_file or self.key key = key_as_s3_uri(bucket=bucket, key=key) - self.init_duckdb() self.duckdb_connection.execute( f""" create or replace table "{table_name}" as ( diff --git a/ostrich_egg/engine.py b/ostrich_egg/engine.py index 6f1543c..7a328eb 100644 --- a/ostrich_egg/engine.py +++ b/ostrich_egg/engine.py @@ -4,35 +4,35 @@ from __future__ import annotations from collections import namedtuple -from copy import deepcopy +from itertools import combinations import os from typing import Dict, List import duckdb from jinja2 import Template -from ostrich_egg.utils import ( - DEFAULT_MASKING_VALUE, - dict_to_filter_expressions, - get_logger, - identifier, - make_when_statement_from_dict, - merge_conditions, -) from ostrich_egg.config import ( + Aggregations, Config, DatasetConfig, - ReplaceWithRedactedParameters, - MergeDimensionValuesParameters, + DimensionOrder, + DEFAULT_THRESHOLD, + load_strategy_from_dict, MarkRedactedParameters, + MergeDimensionValuesParameters, Metric, - Aggregations, + ReplaceWithRedactedParameters, Strategy, - load_strategy_from_dict, - DEFAULT_THRESHOLD, ) from ostrich_egg.connectors import Connector, DEFAULT_TABLE_NAME from ostrich_egg.connectors.s3 import key_as_s3_uri +from ostrich_egg.utils import ( + DEFAULT_MASKING_VALUE, + get_logger, + identifier, + make_when_statement_from_dict, + ostrich_egg_jinja_env, +) DEFAULT_METRIC = "count(*)" @@ -350,270 +350,116 @@ def drop_dimension(self, dimension: str): ) self.removed_dimensions.append(self.active_dimensions.pop(dimension)) - def get_dimension_values_to_redact_with_latency_check( + def redact_from_non_anonymous_cells( self, dimension: str, masking_value=DEFAULT_MASKING_VALUE, - ) -> List[RedactionIterationResult]: + non_summable_dimensions: list[str] = [], + first_order_only: bool = False, + ): """ - The goal here is to flag which rows based on dimension value - need to be marked `redacted`; it will of course by the small value, but it also needs to be - adjacent values until revelation by subtraction (or latent revelation) is not possible. - - For example, if the dimension we're checking is `race` and there are 4 values: - * white: 100 - * black: 50 - * asian: 20 - * native_am: 10 + Iteratively suppress adjacent cells according to dataset/suppression strategy configuration. - then we'd need to set `native_am` to `Redacted` but also `asian` to `Redacted` when strategy is `redact` + The `dimension` is the intended target of suppression; this is most-relevant in upstream processes + in which you need to redact across several dimensions. - If strategy is `merge`, construct a new value that is alpha-sort the redacted values, in this case `asian and native_am`. - The merge strategy without constraint runs the risk of revelation through version iteration; if someone archived the results of this dataset in the past and the - merging is dynamic, then theoretically the difference in versions could reveal a historic data point. + The strategy is to find non-anonymous cells (those that met the redaction expression criteria) and sort the dataset + in a window partitioned by each combination of dimensions and ordering by the redacted dimension within the window (deferring to other sorting configurations first). - It would be required to ensure that the same merged value was used in subsequent iterations so that it can only become more obfuscating and not allow for finer-detailed separation by subtraction. - In this example, it would be necessary for the subsequent iterations of this dataset to not include `asian` and `native_am` but the merged `asian and native_am` value. + We then iteratively suppress the output until the conditions for anonymity are met by virtue of flagging the cells to redact. """ masking_value = masking_value or DEFAULT_MASKING_VALUE - final_result = [] - dimensions_to_hold_constant = [ - d for d in self.active_dimensions if d != dimension - ] - - sql_expressions = self.dimensions_as_sql_expressions( - dimensions=dimensions_to_hold_constant + dimension_sets_to_check = sorted( + [ + dimension_set + for i in range(len(self.active_dimensions)) + for dimension_set in combinations(self.active_dimensions, i + 1) + ], + key=lambda x: len(x), + reverse=True, ) - metrics_sql = ", ".join(self.get_metric_sql_list(initial=False)) - metric_sort = ", ".join(self.get_metric_aliases(initial=False).keys()) - anonymous_metric = "count(*) filter (where not is_anonymous) = 0" - dimension_value_count_sql = f""" - select dense_rank() over(order by {sql_expressions.aliases}) as peer_id , * - from ( - select {sql_expressions.select}, {identifier(dimension)} as dimension_value, {metrics_sql}, {anonymous_metric} as is_anonymous - from result - group by {sql_expressions.group_by}, {identifier(dimension)} - ) as re_agg /* in case we have already dropped columns, we need to reaggregate. */ - order by peer_id, {metric_sort} - """ - identify_peers = self.connector.duckdb_connection.sql(dimension_value_count_sql) - self.connector.duckdb_connection.register("identify_peers", identify_peers) - - # get smallest cells first. - # unless the user wants to prioritize say, time and/or larger geographic units or other semantically useful sort conditions. - order_peers_by = f"{IS_ANONYMOUS_COLUMN}, {metric_sort}, peer_id" - if self.active_dataset.redaction_order_dimensions: - order_by_dimensions = ", ".join( - [ - d - for d in self.active_dataset.redaction_order_dimensions - if d in dimensions_to_hold_constant - ] - ) - order_peers_by = f"{order_by_dimensions}, {order_peers_by}" - peer_ids = [ - x[0] - for x in self.connector.duckdb_connection.sql( - f""" - select distinct peer_id - from identify_peers - order by {order_peers_by} - """ - ).fetchall() - ] - # this variable goes across peers - # it will be set to True if a small cell needs a neighbor to be suppressed. - must_anonymize_next = False - reason = None - for peer_id in peer_ids: - # identify the peers within this aggregation that might need to be latently redacted - peer_value_sql = f""" - select x as peer - from ( - select {sql_expressions.aliases} - from identify_peers - where peer_id = $peer_id - group by {sql_expressions.aliases} - ) as x - """ - peer_values, *_ = self.connector.duckdb_connection.sql( - query=peer_value_sql, params={"peer_id": peer_id} - ).fetchone() - - filter_by_peer = duckdb.ColumnExpression("peer_id").isin(peer_id) - # within a peer group, identify the smallest cells first. - this_peer_relation = ( - identify_peers.filter(filter_by_peer) - .select( - DIMENSION_VALUE_COLUMN, - IS_ANONYMOUS_COLUMN, - *self.get_metric_aliases(initial=False).keys(), - ) - .order( - f"{IS_ANONYMOUS_COLUMN}, {metric_sort}, dimension_value nulls last" - ) - ) - peer_result, must_anonymize_next = ( - self._collect_redactions_from_peer_relation( - this_peer_relation=this_peer_relation, - peer_values=peer_values, - masking_value=masking_value, - must_anonymize_next=must_anonymize_next, - reason=reason, - ) - ) - if must_anonymize_next: - reason = peer_result[-1].reason - else: - reason = None - final_result.extend(peer_result) - - return final_result - - def _collect_redactions_from_peer_relation( - self, - this_peer_relation: duckdb.DuckDBPyRelation, - peer_values: dict, - must_anonymize_next: bool = False, - masking_value: str = DEFAULT_MASKING_VALUE, - reason: str | None = None, - ) -> tuple[list[RedactionIterationResult], bool]: - """ - Main engine for latent anonymization - - Assumes `this_peer_relation` has 2 values: dimension_value and metric_value - and is sorted ascending by metric_value. - Returns the redactions for this group and whether we need to anonymize the next batch. - If it's the last batch and we need anonymizing we've just redacted everything. - """ - values_to_mask = [] - seen_values = [] - - row_count, *_ = this_peer_relation.count("*").fetchone() - within_peer_index = 0 - values_meeting_redaction_criteria = [] - while within_peer_index < row_count: - local_result = this_peer_relation.fetchone() - if not local_result: - break - local_result = dict(zip(this_peer_relation.columns, local_result)) - value_is_fine = local_result[IS_ANONYMOUS_COLUMN] - dimension_value = local_result[DIMENSION_VALUE_COLUMN] - seen_values.append(dimension_value) - - if within_peer_index > 0: - working_subtotal_query_template = Template( - """\ - select x - from ( - select *, {{anonymous_expression}} as {{is_anonymous}} - from ( - select {{metric_list|join(', ')}} - from identify_peers - where list_contains($seen_values, dimension_value) - ) as aggregated - ) as x - """ - ) - sql = working_subtotal_query_template.render( - metric_list=self.get_metric_sql_list(initial=False), - anonymous_expression=self.anonymous_expression, - is_anonymous=IS_ANONYMOUS_COLUMN, - ) - working_subtotal, *_ = self.connector.duckdb_connection.sql( - sql, params={"seen_values": seen_values} - ).fetchone() - working_total_is_fine = working_subtotal[IS_ANONYMOUS_COLUMN] - else: - working_total_is_fine = value_is_fine + check_redacted_context_template = ostrich_egg_jinja_env.get_template( + "check_redacted_context.sql" + ) - first_value_is_good = within_peer_index == 0 and value_is_fine + check_redacted_context_sql = check_redacted_context_template.render( + non_summable_dimensions=non_summable_dimensions, + threshold=self.threshold, + incidence_column=self.metrics[0].alias, + first_order_only=first_order_only, + ) - sufficient_prior_redaction = ( - len(values_to_mask) >= 2 and working_total_is_fine + update_output_from_redaction_context_template = ( + ostrich_egg_jinja_env.get_template( + "update_output_from_redaction_context.sql" ) + ) - if sufficient_prior_redaction: - must_anonymize_next = False - - if not must_anonymize_next and first_value_is_good: - """ - The first value in the peer group is already anonymized and we don't need to anonymize from a previous peer group, this is anonymous already. - """ - break - - elif not value_is_fine: - """ - Base case small-cell for suppression. - """ - - logger.debug( - f"{dimension_value} meets redaction criteria\n {self.redaction_expression}" - ) - values_meeting_redaction_criteria.append(dimension_value) - values_to_mask.append(dimension_value) - must_anonymize_next = True - - elif must_anonymize_next and value_is_fine: - """ - Have to anonymize based on a previous redaction. - """ - values_to_mask.append(dimension_value) - must_anonymize_next = False - break - - elif not sufficient_prior_redaction or must_anonymize_next: - """ - This is the base case for latent suppression in which the value itself is fine but we must suppress to prevent latent revelation. - We will check the next iteration if the redaction leaves us with exclusively sufficiently large cells and no additional revelation through subtraction. - """ - logger.debug( - f"Redacting {dimension_value} as it would latently reveal an unacceptable metric value through subtraction." - ) - values_to_mask.append(dimension_value) - - # If this is the last value in the peer group and we are _now_ anonymized, we do not need to anonymize the next peer group. - # else we do not yet have sufficient prior redaction to prevent latent revelation, and we must suppress another cell. - if len(values_to_mask) >= 2 and ( - working_total_is_fine or value_is_fine - ): - must_anonymize_next = False - - elif sufficient_prior_redaction and value_is_fine: - """ - This is an exit condition from when redaction is required; there is enough small cell suppression - and the values in this peer group from this value forward are all fine ( it was sorted by value ascending). - """ - must_anonymize_next = False - break - - within_peer_index += 1 - if values_meeting_redaction_criteria: - value_string = ", ".join( - [ - "" if value is None else str(value) - for value in values_meeting_redaction_criteria - ] + update_output_from_redaction_context_sql = ( + update_output_from_redaction_context_template.render( + dimensions=self.active_dimensions, + output_table="output", + threshold=self.threshold, ) + ) - reason = f"value{'s' if len(values_meeting_redaction_criteria) > 1 else ''} {value_string} meet{'s' if len(values_meeting_redaction_criteria) == 1 else ''} redaction criteria\n {self.redaction_expression}" - peer_result = ( - [ - RedactionIterationResult( - other_dimension_values=peer_values, - remapped_lookup={ - dimension_value: masking_value - for dimension_value in values_to_mask - }, - reason=reason, - ) + for dimension_set in dimension_sets_to_check: + dimension_set_order_by_columns = [ + DimensionOrder(dimension=dimension) for dimension in dimension_set ] - if values_to_mask - else [] - ) - return peer_result, must_anonymize_next + order_by_columns = [ + # small cells on top + DimensionOrder(dimension="is_anonymous", direction="asc"), + # followed by already redacted ones + DimensionOrder(dimension="is_redacted", direction="desc"), + # then user configuration for prioritization + *self.active_dataset.redaction_order_dimensions, + # then sort within the dimension set window + *[ + dim + for dim in dimension_set_order_by_columns + if dim.dimension + not in self.active_dataset.redaction_order_dimensions + ], + # then sort by the dimension to redact + DimensionOrder(dimension=dimension), + # with smallest cells above bigger cells. + DimensionOrder(dimension=self.metrics[0].alias), + ] + + redaction_context_view_template = ostrich_egg_jinja_env.get_template( + "redaction_context_view.sql" + ) + redaction_context_sql = redaction_context_view_template.render( + dimension_set=dimension_set, + order_by_columns=order_by_columns, + dimension=dimension, + non_summable_dimensions=non_summable_dimensions, + output_table="output", + incidence_column=self.metrics[0].alias, + active_dimensions=self.active_dimensions, + ) + logger.info(f"Creating redaction context view for {dimension_set}") + logger.debug(redaction_context_sql) + redaction_context_view = self.db.sql(redaction_context_sql) + self.db.register("redaction_context", redaction_context_view) + + logger.info("Looking for records to redact") + logger.debug(check_redacted_context_sql) + + to_redact = self.db.sql(check_redacted_context_sql) + self.db.register("to_redact", to_redact) + to_redact_count = to_redact.count("*").fetchone()[0] + while to_redact_count > 0: + logger.info(f"Found {to_redact_count} records to redact") + logger.debug(to_redact.to_df().to_json(orient="records", indent=2)) + self.db.execute(update_output_from_redaction_context_sql) + + to_redact = self.db.sql(check_redacted_context_sql) + self.db.register("to_redact", to_redact) + to_redact_count = to_redact.count("*").fetchone()[0] def update_the_dataset( self, dimension, existing_values: list, new_value: str = "Redacted" @@ -642,107 +488,70 @@ def merge_dimension_values(self, parameters: List[MergeDimensionValuesParameters ) ) - def replace_with_redacted(self, params: ReplaceWithRedactedParameters): - """ - Sets the update expression to be used in the final result by assigning redaction - values to dimensions. - """ - # TODO: add in the non-summable dimensions? - redactions = self.get_dimension_values_to_redact_with_latency_check( - dimension=params.redacted_dimension, - masking_value=params.masking_value, - ) - - self.redactions[params.redacted_dimension] = self.redactions.get( - params.redacted_dimension, [] - ) - self.redactions[params.redacted_dimension].extend(redactions) - self.run_aggregation() - self.connector.duckdb_connection.execute( - "create or replace table output as select * from result" - ) - - def mark_redacted(self, params: MarkRedactedParameters): - """ - Adds a column `is_redacted` which is either true (for the value appears in the redactions) or false if the column is usable. - This is useful for when you need the source values for your process but need to indicate which cells - must be suppressed. - """ + def modify_output_for_redaction(self): self.connector.duckdb_connection.execute( "create or replace table output as select * from result" ) - result_table = self.connector.duckdb_connection.table("output") - - dimension_to_use = params.redacted_dimension - redactions = [] - if params.non_summable_dimensions: - columns = [identifier(dim) for dim in params.non_summable_dimensions] - pages = ( - result_table.select(*columns) - .distinct() - .to_df() - .to_dict(orient="records") - ) - for page in pages: - filters = dict_to_filter_expressions(data=page) - condition = merge_conditions(filters) - result = result_table.filter(condition) - self.connector.duckdb_connection.register("result", result) - page_redactions = ( - self.get_dimension_values_to_redact_with_latency_check( - dimension=dimension_to_use - ) - ) - redactions.extend(page_redactions) - else: - self.connector.duckdb_connection.register("result", result_table) - redactions = self.get_dimension_values_to_redact_with_latency_check( - dimension=dimension_to_use, - ) alter_sql = """ alter table output add column "is_redacted" boolean default false; alter table output - add column "peer_group" json; + add column "peer_group" json[]; alter table output - add column "redacted_peers" json; + add column "redacted_peers" json[]; alter table output add column "redaction_reason" text; """ self.connector.duckdb_connection.execute(alter_sql) self.connector.duckdb_connection.execute( - "update output set is_redacted = true where not is_anonymous" - ) - for redaction in redactions: - for old, new in redaction.remapped_lookup.items(): - peer_group = deepcopy(redaction.other_dimension_values) - redaction.other_dimension_values[dimension_to_use] = old - filters = dict_to_filter_expressions( - data=redaction.other_dimension_values - ) - condition = merge_conditions(filters) - update_sql = f""" + f"""\ update output - set "is_redacted" = true, - "redaction_reason" = $redaction_reason, - "peer_group" = $peer_group, - "redacted_peers" = $redacted_peers - where {condition} - """ - if peer_group and not redaction.reason: - logger.error( - f"{redaction.other_dimension_values=} {redaction.remapped_lookup=}" - ) - self.connector.duckdb_connection.execute( - query=update_sql, - parameters={ - "redaction_reason": redaction.reason, - "peer_group": peer_group, - "redacted_peers": { - dimension_to_use: list(redaction.remapped_lookup.keys()) - }, - }, - ) + set is_redacted = true + , redaction_reason = $$value meets redaction criteria \n'{self.redaction_expression}'$$ + where not is_anonymous + """ + ) + + def update_output_json_types(self): + self.db.execute( + """\ + alter table output alter column "peer_group" type json using peer_group::json[]::json; + alter table output alter column "redacted_peers" type json using redacted_peers::json[]::json; + """ + ) + + def replace_with_redacted(self, params: ReplaceWithRedactedParameters): + """ + Sets the update expression to be used in the final result by assigning redaction + values to dimensions. + """ + self.mark_redacted(params=params) + self.db.execute( + f"""\ + update output + set {params.redacted_dimension} = case + when is_redacted then '{params.masking_value}' + else {params.redacted_dimension} + end + where is_redacted + """ + ) + self.update_output_json_types() + self.final_source_table = "output" + + def mark_redacted(self, params: MarkRedactedParameters): + """ + Adds a column `is_redacted` which is either true (for the value appears in the redactions) or false if the column is usable. + This is useful for when you need the source values for your process but need to indicate which cells + must be suppressed. + """ + self.modify_output_for_redaction() + self.redact_from_non_anonymous_cells( + dimension=params.redacted_dimension, + non_summable_dimensions=params.non_summable_dimensions or [], + first_order_only=params.first_order_only, + ) + self.update_output_json_types() # Use this new table instead of the source data. self.final_source_table = "output" @@ -811,7 +620,6 @@ def run(self, output_file: str = None): """ Read configs passed to the engine and processes the dataset to produce output accordingly. """ - self.connector.init_duckdb() for index, dataset in enumerate(self.datasets): self.run_one_dataset(index=index, dataset=dataset, output_file=output_file) diff --git a/ostrich_egg/templates/__init__.py b/ostrich_egg/templates/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ostrich_egg/templates/check_redacted_context.sql b/ostrich_egg/templates/check_redacted_context.sql new file mode 100644 index 0000000..b5b9f0a --- /dev/null +++ b/ostrich_egg/templates/check_redacted_context.sql @@ -0,0 +1,23 @@ +{# +Query the redaction context to find cells that need suppressed to prevent latent revelation. + #} +select * +from redaction_context +where + not is_redacted + and should_redact_along_axis( + incidence := {{ incidence_column | identifier }}, + masked_value_count := masked_value_count, + minimum_threshold := {{ threshold }}, + is_anonymous := is_anonymous, + previous_cell_redacted := previous_cell_redacted, + previous_cell_is_anonymous := previous_cell_is_anonymous, + run_sum_by_axis := run_sum_by_axis, + first_order_only := {{ first_order_only | default(False) }} + ) + {#- If any dimensions are not aggregable, i.e., users won't know the total sum of these dimensions, + then we just need to only consider redacting values where those dimensions match. + + Else, this might consider other peer-groups to redact along these dimensions. + -#} + {%- for dim in non_summable_dimensions %} and {{ dim | identifier }} = "previous_{{ dim | dequote }}" {% endfor %} diff --git a/ostrich_egg/templates/redaction_context_view.sql b/ostrich_egg/templates/redaction_context_view.sql new file mode 100644 index 0000000..f0ee7ac --- /dev/null +++ b/ostrich_egg/templates/redaction_context_view.sql @@ -0,0 +1,53 @@ +{# +Generates the context needed to evaluate which cells need to be suppressed to avoid latent revelation. + +For each each combination of dimensions, the calling method `redact_from_non_anonymous_cells` +passes configurations for sorting and partitioning the dataset such that we find latent revelation through iterative windows. + #} +{%- set partition_and_order_by -%} +over( partition by {{dimension_set | list_of_identifiers}} order by {{order_by_columns | map(attribute='sql_expression') | join(', ')}}) +{%- endset -%} +{%- set peer_group_expression -%} +{ {#- -#} +{% for dim in dimension_set %} + {{ dim | identifier }}:{{ dim | identifier }}{% if not loop.last %}, {% endif %} +{% endfor %} +{#- -#} } +{%- endset -%} + +{%- set previous_row_expression -%} +{ {#- -#} +{% for dim in active_dimensions %} + {{ dim | identifier }}:{{ dim | identifier }}{% if not loop.last %}, {% endif %} +{% endfor %} +{#- -#} } +{%- endset -%} + +{%- set redacted_peers_expression -%} +{ {#- -#} +{{ dimension | identifier }}:{{ dimension | identifier }} +{#- -#} } +{%- endset -%} +select + * + {#- Duckdb star expression `replace` replaces the newly created output columns with the windowed response. #} + replace( + ({{ peer_group_expression }})::json as peer_group, ({{ redacted_peers_expression }})::json as redacted_peers, + ), + (lag({{ peer_group_expression }}) {{ partition_and_order_by }})::json as previous_peer_group, + (lag({{ redacted_peers_expression }}) {{ partition_and_order_by }})::json as previous_redacted_peers, + + lag(is_redacted) {{ partition_and_order_by }} as previous_cell_redacted, + lag(is_anonymous) {{ partition_and_order_by }} as previous_cell_is_anonymous, + lag(redaction_reason) {{ partition_and_order_by }} as previous_cell_redaction_reason, + lag({{ incidence_column | identifier }}) {{ partition_and_order_by }} as previous_incidence, + sum({{ incidence_column | identifier }}) {{ partition_and_order_by }} as run_sum_by_axis, + count(*) filter(where is_redacted) over ( + partition by {{ dimension_set | list_of_identifiers }} + ) as masked_value_count, + lag({{previous_row_expression}}) {{ partition_and_order_by }} as previous_row + {%- for dim in non_summable_dimensions %} + , lag({{ dim | identifier }}) {{ partition_and_order_by }} as "previous_{{ dim | dequote }}" + {%- endfor %}, + +from {{ output_table | default("output") }} as "output" diff --git a/ostrich_egg/templates/update_output_from_redaction_context.sql b/ostrich_egg/templates/update_output_from_redaction_context.sql new file mode 100644 index 0000000..2bc51ff --- /dev/null +++ b/ostrich_egg/templates/update_output_from_redaction_context.sql @@ -0,0 +1,15 @@ +update {{output_table|default("output")}} as "output" +set is_redacted = true +, redacted_peers = list_distinct(flatten([[to_redact.redacted_peers, to_redact.previous_redacted_peers], coalesce("output".redacted_peers, [])])) +, peer_group = list_distinct(flatten([[to_redact.peer_group, to_redact.previous_peer_group], coalesce("output".peer_group, [])])) +, redaction_reason = case + when output.redaction_reason is not null then output.redaction_reason + when not to_redact.previous_cell_is_anonymous then format('{0} was a small cell', previous_row::json ) + when masked_value_count < 2 then previous_cell_redaction_reason + when run_sum_by_axis - previous_incidence < {{ threshold }} then previous_cell_redaction_reason || ' and the delta would construct a small population.' + +end +from to_redact +where {% for dim in dimensions -%} + {% if not loop.first %} and {% endif %}coalesce("output".{{ dim | identifier }}::text, '') = coalesce(to_redact.{{ dim | identifier }}::text, '') +{% endfor %} diff --git a/ostrich_egg/utils.py b/ostrich_egg/utils.py index 19c780c..50a7cf7 100644 --- a/ostrich_egg/utils.py +++ b/ostrich_egg/utils.py @@ -1,8 +1,10 @@ import os import logging -import duckdb from typing import List +import duckdb +from jinja2 import Environment, PackageLoader + ENVIRONMENT_LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO") DEFAULT_MASKING_VALUE = "redacted" @@ -63,3 +65,60 @@ def make_when_statement_from_dict(data: dict, value: str) -> duckdb.CaseExpressi condition = merge_conditions(filters) expression = f"when {condition} then {value_expression}" return expression + + +def should_redact_along_axis( + incidence: float | int, + masked_value_count: int = 0, + minimum_threshold: int | float = 11, + is_anonymous: bool = True, + previous_cell_redacted: bool | None = None, + previous_cell_is_anonymous: bool | None = None, + run_sum_by_axis: float | int = 0, + first_order_only: bool = False, +) -> bool: + """ + Given a set of dimensions (i.e., the axis) and some pre-calculated windowed rows, + determine if the cell needs to be redacted based on the available criteria. + + This will be run iteratively until the dataset consists only of cells that are suppressed to anonymity. + + The first_order_only flag will not redact cells if the previous cell in the window + was suppressed to redact along a different dimension. + + This reduces the total amount of redaction and thus should be used cautiously as it _technically_ allows a very determined + actor to reveal the small cell by finding the methodically calculating the revelation-through-subtraction of other dimensional combinations. + + If your use case already protects individuals and is only to implement a requirement not to display small cells or reveal through first-order subtraction, + then this can suppress small cells along each dimension that _the small cell_ could be revealed and does not redact other suppressed cells ensuring true anonymization.. + """ + if not is_anonymous: + return True # all non-anonymous cells need redacted. + if previous_cell_redacted is False: + return False # there is no latency in this pass, do not redact. + + # if previous_cell_redacted is True + + if run_sum_by_axis - incidence >= minimum_threshold: + if first_order_only: + # we only need to suppress along the "view" of the dimensional axis, + # we're not suppressing suppressed cells only. + return previous_cell_is_anonymous is False and masked_value_count < 2 + else: + # default: we are suppressing along each axis and have to suppress all redacted cells + # so as not to enable revelation through subtraction across views. + return masked_value_count < 2 + else: + return True + + +ostrich_egg_jinja_env = Environment( + loader=PackageLoader("ostrich_egg"), +) +ostrich_egg_jinja_env.filters.update( + { + "identifier": identifier, + "dequote": lambda x: x.replace('"', ""), + "list_of_identifiers": lambda x: ", ".join([identifier(i) for i in x]), + } +) diff --git a/pyproject.toml b/pyproject.toml index 7ac08aa..7fb1485 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,19 +1,23 @@ [project] -name = "ostrich-egg" -version = "0.0.0" -description = "A tool for producing public analytics while protecting data privacy." -readme = "README.md" +name = "ostrich-egg" +version = "0.0.0" +description = "A tool for producing public analytics while protecting data privacy." +readme = "README.md" dependencies = ["boto3", "duckdb", "jinja2", "pydantic"] -authors = [{ name = "Jacob Hickson", email = "jhickson@greenriver.org" }] +authors = [{ name = "Jacob Hickson", email = "jhickson@greenriver.org" }] [build-system] -requires = ["setuptools"] +requires = ["setuptools"] build-backend = "setuptools.build_meta" [tool.pytest.ini_options] -testpaths = "tests" -addopts = "--tb=auto -vv --log-level=ERROR --log-cli-level=ERROR --show-capture=stderr" +testpaths = "tests" +addopts = "--tb=auto -vv --log-level=ERROR --log-cli-level=ERROR --show-capture=stderr" filterwarnings = ["ignore::DeprecationWarning", "ignore::FutureWarning"] +timeout = 30 [tool.setuptools.packages.find] exclude = ["tests", "dependencies"] + +[tool.setuptools.package-data] +"*" = ["templates/*.sql"] diff --git a/schemas/config_schema.json b/schemas/config_schema.json new file mode 100644 index 0000000..4542f73 --- /dev/null +++ b/schemas/config_schema.json @@ -0,0 +1,709 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$defs": { + "Aggregations": { + "enum": [ + "any_value", + "array_agg", + "avg", + "count", + "count_distinct", + "max", + "min", + "sum" + ], + "title": "Aggregations", + "type": "string" + }, + "DataSource": { + "properties": { + "connection_type": { + "description": "The type of connection to use for this data source. Must be one of \"s3\" \"file\" or \"postgres.", + "enum": [ + "s3", + "file" + ], + "title": "Connection Type", + "type": "string" + }, + "parameters": { + "anyOf": [ + { + "description": "Connection parameters for the data source", + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Parameters" + } + }, + "required": [ + "connection_type" + ], + "title": "DataSource", + "type": "object" + }, + "DatasetConfig": { + "description": "Service configuration", + "properties": { + "name": { + "anyOf": [ + { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "A human-friendly way to name this dataset" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" + }, + "dimensions": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "A list of dimensions that should are included in the dataset. These are the actual column headers. If null, then all columns excluding the unit-level-id and the metrics will be considered dimensions.", + "title": "Dimensions" + }, + "unit-level-id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The column name for the unit-level-id, e.g., not-aggregate but identifies a unique record. This will be ignored in calculations except for count(distinct unit-level-id) and the metrics will group by the dimensions.", + "title": "Unit-Level-Id" + }, + "initial_metrics": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Metric" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An initial set of metrics that will be produced from the first aggregation of the dataset; if not specified and no unit-level-id, will just use count(*). If unit-level-id, will use count(distinct unit-level-id).", + "title": "Initial Metrics" + }, + "metrics": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Metric" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The metrics that will be produced when aggregation is run. If not specified and no unit-level-id, will use count(*). If unit-level-id, will use count(distinct unit-level-id). If only count(*) is specified, a metric will be added to the engine to run subsequently to sum the count metric.", + "title": "Metrics" + }, + "sql": { + "anyOf": [ + { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The internal SQL to produce the 'view' of this dataset" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sql" + }, + "redaction_order_dimensions": { + "anyOf": [ + { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/DimensionOrder" + }, + "type": "array" + }, + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The dimensions to order the redaction by. If not specified, the dimensions will be ordered by the order they are specified in the dataset ascending." + }, + { + "type": "null" + } + ], + "title": "Redaction Order Dimensions" + }, + "source_file": { + "anyOf": [ + { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "At present this is called a file for simplicity. It's actually likely a key for s3 or a fully qualified table name for postgres. If not specified, the source is in the datasource." + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source File" + }, + "suppression-strategies": { + "anyOf": [ + { + "items": { + "discriminator": { + "mapping": { + "mark-redacted": "#/$defs/MarkRedacted", + "merge-dimension-values": "#/$defs/MergeDimensionValuesStrategy", + "reduce-dimensions": "#/$defs/ReduceDimensionsStrategy", + "replace-with-redacted": "#/$defs/ReplaceWithRedacted" + }, + "propertyName": "strategy" + }, + "oneOf": [ + { + "$ref": "#/$defs/ReduceDimensionsStrategy" + }, + { + "$ref": "#/$defs/MergeDimensionValuesStrategy" + }, + { + "$ref": "#/$defs/ReplaceWithRedacted" + }, + { + "$ref": "#/$defs/MarkRedacted" + } + ] + }, + "type": "array" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of suppression strategies", + "title": "Suppression-Strategies" + }, + "output_file": { + "anyOf": [ + { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The filepath you want want to write out to. In the future, this can be more of an object that can configure things like table, s3 location, etc, but for Proof-of-concept just writing to file (which is really an s3 key)" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Output File" + }, + "extra": { + "anyOf": [ + { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Dataset-level configurations for your application. For example, if you need to map the results of intermediary dimensionality." + }, + { + "type": "null" + } + ], + "default": null, + "title": "Extra" + } + }, + "required": [ + "dimensions" + ], + "title": "DatasetConfig", + "type": "object" + }, + "DimensionOrder": { + "properties": { + "dimension": { + "title": "Dimension", + "type": "string" + }, + "direction": { + "default": "asc", + "enum": [ + "asc", + "desc" + ], + "title": "Direction", + "type": "string" + } + }, + "required": [ + "dimension" + ], + "title": "DimensionOrder", + "type": "object" + }, + "MarkRedacted": { + "description": "Simply flag cells that must be suppressed by virtue of values under threshold or by virtue of requirement through latent revelation.", + "properties": { + "strategy": { + "const": "mark-redacted", + "default": "mark-redacted", + "title": "Strategy", + "type": "string" + }, + "parameters": { + "anyOf": [ + { + "$ref": "#/$defs/MarkRedactedParameters" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "title": "MarkRedacted", + "type": "object" + }, + "MarkRedactedParameters": { + "properties": { + "redacted_dimension": { + "description": "The dimension to check for redaction. The output's `is_redacted` flag will apply to this dimension. One iteration will only produce cell-level suppression and applies only to this dimension. You must iterate for other dimensions to achieve full dataset anonymization.", + "title": "Redacted Dimension", + "type": "string" + }, + "non_summable_dimensions": { + "anyOf": [ + { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of dimensions that are part of the dataset but will not ever actually be aggregated. For example, if you have unrelated indicators in a column or years when you won't ever sum the total number of incidences across time (preventing revelation through subtraction). Use with caution." + }, + { + "type": "null" + } + ], + "default": null, + "title": "Non Summable Dimensions" + }, + "first_order_only": { + "anyOf": [ + { + "default": false, + "description": "Whether to only redact cells to prevent latent revelation along a single axis of dimensions. Enabling reduces the total number of cells suppressed but creates a risk of transitive revelation across dimensions.", + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "title": "First Order Only" + } + }, + "required": [ + "redacted_dimension" + ], + "title": "MarkRedactedParameters", + "type": "object" + }, + "MergeDimensionValuesParameters": { + "properties": { + "dimension": { + "description": "The dimension whose values are to be merged", + "title": "Dimension", + "type": "string" + }, + "values": { + "description": "The ordinal precedence for which values to merge into one another", + "items": { + "type": "string" + }, + "title": "Values", + "type": "array" + }, + "merged_value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The name of the value to merge the values into.", + "title": "Merged Value" + } + }, + "required": [ + "dimension", + "values" + ], + "title": "MergeDimensionValuesParameters", + "type": "object" + }, + "MergeDimensionValuesStrategy": { + "properties": { + "strategy": { + "const": "merge-dimension-values", + "title": "Strategy", + "type": "string" + }, + "parameters": { + "description": "The list of dimensions with their values and merged values to apply.", + "items": { + "$ref": "#/$defs/MergeDimensionValuesParameters" + }, + "title": "Parameters", + "type": "array" + } + }, + "required": [ + "strategy", + "parameters" + ], + "title": "MergeDimensionValuesStrategy", + "type": "object" + }, + "Metric": { + "properties": { + "aggregation": { + "$ref": "#/$defs/Aggregations", + "description": "What aggregation to use for this metric. one of ['any_value', 'array_agg', 'avg', 'count', 'count_distinct', 'max', 'min', 'sum'] " + }, + "column": { + "description": "The column name this metric wraps around. if pre-aggregated, just use sum (assuming you have correctly flagged the dimensions to aggregate). Use star for COUNT ", + "title": "Column", + "type": "string" + }, + "alias": { + "anyOf": [ + { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Alias for this metric to use as a name for reporting it. Not strictly necessary but might make reading the report easier if you have multiple metrics." + }, + { + "type": "null" + } + ], + "default": null, + "title": "Alias" + }, + "null_is_zero": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "If true, null values will be treated as 0 via coalesce. If false, null values will be discarded in the result of the aggregation.", + "title": "Null Is Zero" + }, + "expression": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A literal expression to use for the metric. Let's a user create advanced custom metrics", + "title": "Expression" + }, + "is_initial": { + "default": false, + "description": "Whether this metric is only an initial metric against the base dataset to produce the initial aggregation before running latent checks. It is by default false.", + "title": "Is Initial", + "type": "boolean" + }, + "is_subsequent": { + "description": "Whether this metric is a subsequent metric that runs after the initial metric. It is by default the opposite of is_initial; a metric could be the same for both, especially if already a a sum or average or any_value aggregation.", + "title": "Is Subsequent", + "type": "boolean" + } + }, + "required": [ + "aggregation", + "column" + ], + "title": "Metric", + "type": "object" + }, + "ReduceDimensionsParameters": { + "properties": { + "dimensions": { + "description": "A prioritized sequential list of the dimensions to 'prune' for re-aggregation; the service will attempt to re-aggregate and re-sample in this order. ", + "items": { + "type": "string" + }, + "title": "Dimensions", + "type": "array" + } + }, + "required": [ + "dimensions" + ], + "title": "ReduceDimensionsParameters", + "type": "object" + }, + "ReduceDimensionsStrategy": { + "properties": { + "strategy": { + "const": "reduce-dimensions", + "title": "Strategy", + "type": "string" + }, + "parameters": { + "$ref": "#/$defs/ReduceDimensionsParameters", + "description": "An object containing parameters for the strategy" + } + }, + "required": [ + "strategy", + "parameters" + ], + "title": "ReduceDimensionsStrategy", + "type": "object" + }, + "ReplaceWithRedacted": { + "properties": { + "strategy": { + "const": "replace-with-redacted", + "default": "replace-with-redacted", + "title": "Strategy", + "type": "string" + }, + "parameters": { + "$ref": "#/$defs/ReplaceWithRedactedParameters" + } + }, + "required": [ + "parameters" + ], + "title": "ReplaceWithRedacted", + "type": "object" + }, + "ReplaceWithRedactedParameters": { + "properties": { + "redacted_dimension": { + "description": "The dimension to check for redaction. The output's `is_redacted` flag will apply to this dimension. One iteration will only produce cell-level suppression and applies only to this dimension. You must iterate for other dimensions to achieve full dataset anonymization.", + "title": "Redacted Dimension", + "type": "string" + }, + "dimensions": { + "description": "A prioritized sequential list of the dimensions to mark small values as `redacted` (or other masking value)", + "items": { + "type": "string" + }, + "title": "Dimensions", + "type": "array" + }, + "masking_value": { + "anyOf": [ + { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "redacted", + "description": "The masking value to apply to values in the dimension." + }, + { + "type": "null" + } + ], + "default": null, + "title": "Masking Value" + }, + "non_summable_dimensions": { + "anyOf": [ + { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "List of dimensions that are part of the dataset but will not ever actually be aggregated. For example, if you have unrelated indicators in a column or years when you won't ever sum the total number of incidences across time (preventing revelation through subtraction). Use with caution." + }, + { + "type": "null" + } + ], + "default": null, + "title": "Non Summable Dimensions" + }, + "first_order_only": { + "anyOf": [ + { + "default": false, + "description": "Whether to only redact cells to prevent latent revelation along a single axis of dimensions. Enabling reduces the total number of cells suppressed but creates a risk of transitive revelation across dimensions.", + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "title": "First Order Only" + } + }, + "required": [ + "redacted_dimension", + "dimensions" + ], + "title": "ReplaceWithRedactedParameters", + "type": "object" + } + }, + "description": "The actual configuration for the service will typically take the shape of a single datasource and multiple datasets.\nItems can callback to variables or anchors.\nThe outlet is assumed to be always the same type for now, we can complicate later.", + "properties": { + "datasource": { + "$ref": "#/$defs/DataSource" + }, + "allow_zeroes": { + "default": true, + "description": "Whether a 0 value counts as below the threshold for evaluating anonymity. By default, this is true, meaning a 0 is considered anonymous. When false, 0 is considered a small number that is masked.", + "title": "Allow Zeroes", + "type": "boolean" + }, + "redaction_expression": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "This expression is used in the aggregation queries to determine if a cell should be redacted. If not specified, then the first metric will be evaluated at the default threshold of 11.", + "title": "Redaction Expression" + }, + "datasets": { + "items": { + "$ref": "#/$defs/DatasetConfig" + }, + "title": "Datasets", + "type": "array" + }, + "threshold": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 11, + "description": "[DEPRECATED]: Single value for a threshold, being replaced by an expression.", + "title": "Threshold" + } + }, + "required": [ + "datasource", + "datasets" + ], + "title": "Config", + "type": "object" +} \ No newline at end of file diff --git a/schemas/generate.py b/schemas/generate.py new file mode 100644 index 0000000..49853e5 --- /dev/null +++ b/schemas/generate.py @@ -0,0 +1,33 @@ +import json +import os +import sys + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) +from ostrich_egg.config import Config +from pydantic import BaseModel + +DIR = os.path.dirname(__file__) + + +def write_model_to_file(model: BaseModel, file_path: str): + with open(file_path, "w") as f: + schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + } | model.model_json_schema() + json.dump(obj=schema, fp=f, indent=2) + print(os.path.abspath(f.name)) + + +def write_config_to_file(directory=DIR): + + for model, file_path in [ + ( + Config, + os.path.join(directory, "config_schema.json"), + ), + ]: + write_model_to_file(model=model, file_path=file_path) + + +if __name__ == "__main__": + write_config_to_file() diff --git a/tests/conftest.py b/tests/conftest.py index b9e0cba..65f8f0d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -33,7 +33,9 @@ @pytest.fixture() -def mocked_s3_res(moto_server): +def mocked_s3_res(moto_server, monkeypatch): + monkeypatch.delenv("AWS_ENDPOINT_URL", raising=False) + monkeypatch.delenv("AIRFLOW_CONN_AWS_DEFAULT", raising=False) with mock_aws(): yield boto3.resource("s3", endpoint_url=moto_server) diff --git a/tests/data_inputs/library_example.csv b/tests/data_inputs/library_example.csv index a1d70e4..675dc0c 100644 --- a/tests/data_inputs/library_example.csv +++ b/tests/data_inputs/library_example.csv @@ -1,5 +1,13 @@ count,age,sex,zip_code,library_friend 3,30,M,00000,Yes -20,30,F,00000,Yes +20,40,F,00000,Yes 25,30,M,00000,No -12,30,F,00000,No +12,40,F,00000,No +13,40,M,00001,Yes +21,40,F,00001,Yes +26,40,M,00001,No +15,40,F,00001,No +14,40,M,00002,Yes +25,40,F,00002,Yes +27,40,M,00002,No +16,40,F,00002,No diff --git a/tests/data_inputs/multi-dimensional-revelation.json b/tests/data_inputs/multi-dimensional-revelation.json new file mode 100644 index 0000000..64d3033 --- /dev/null +++ b/tests/data_inputs/multi-dimensional-revelation.json @@ -0,0 +1,44 @@ +[ + { + "county": "A", + "month": "2025-01", + "incidence": 10, + "expected_to_be_redacted": true, + "note": "Small cell, gets redacted." + }, + { + "county": "B", + "month": "2025-01", + "incidence": 15, + "expected_to_be_redacted": true, + "note": "County dimension causes suppression of this cell." + }, + { + "county": "C", + "month": "2025-01", + "incidence": 16, + "expected_to_be_redacted": false, + "note": "This cell is not suppressed." + }, + { + "county": "A", + "month": "2025-02", + "incidence": 12, + "expected_to_be_redacted": true, + "note": "This cell won't be suppressed by county, but it needs to be suppressed by month when grouping by county." + }, + { + "county": "B", + "month": "2025-02", + "incidence": 17, + "expected_to_be_redacted": true, + "note": "To prevent latent revelation, this would also need to be suppressed so as not to be able to work systematically to the originally suppressed cell by different totals." + }, + { + "county": "C", + "month": "2025-02", + "incidence": 18, + "expected_to_be_redacted": false, + "note": "This will not be suppressed." + } +] diff --git a/tests/data_inputs/read_only_compound_population_example.csv b/tests/data_inputs/read_only_compound_population_example.csv index 8bb6895..6009e8b 100644 --- a/tests/data_inputs/read_only_compound_population_example.csv +++ b/tests/data_inputs/read_only_compound_population_example.csv @@ -4,4 +4,4 @@ F,00000,True,True,12,10000 F,00001,True,False,22,20000 M,00002,True,False,15,10000 M,00003,False,True,15,1000 -M,00001,True,True,2,20000 +M,00001,True,False,2,20000 diff --git a/tests/data_inputs/redaction_examples.json b/tests/data_inputs/redaction_examples.json index 1b93d68..e0d77ca 100644 --- a/tests/data_inputs/redaction_examples.json +++ b/tests/data_inputs/redaction_examples.json @@ -46,8 +46,8 @@ "zip_code": "zip-code-20000", "incidence": 60, "population_value": 8000, - "expected_to_be_redacted": false, - "purpose": "This is a big cell in a largely suppressed peer group." + "expected_to_be_redacted": true, + "purpose": "This will get suppressed due to the small cell in the peer group." }, { "incidence": 1, diff --git a/tests/data_inputs/redaction_order_config.yml b/tests/data_inputs/redaction_order_config.yml new file mode 100644 index 0000000..21872f3 --- /dev/null +++ b/tests/data_inputs/redaction_order_config.yml @@ -0,0 +1,51 @@ +datasource: + connection_type: file + parameters: + output_directory: /tmp/ +allow_zeroes: true +datasets: + - name: test_high_dimensional_config + dimensions: + - year + - month + - zip_code + metrics: + - aggregation: sum + column: incidence + alias: incidence + is_initial: true + is_subsequent: true + - aggregation: sum + alias: population_value + column: population_value + is_initial: true + is_subsequent: true + - aggregation: any_value + alias: population_value + column: population_value + is_initial: true + is_subsequent: true + - aggregation: any_value + alias: expected_to_redact + column: expected_to_redact + is_initial: true + is_subsequent: true + redaction_order_dimensions: + - dimension: year + direction: asc + - dimension: month + direction: asc + - dimension: population_value + direction: asc + suppression_strategies: + - parameters: + first_order_only: true + non_summable_dimensions: null + redacted_dimension: zip_code + strategy: mark-redacted +redaction_expression: > + case + when population_value < 2000 then true + when incidence < 11 then true else false + end +threshold: 11 diff --git a/tests/data_inputs/redaction_order_config_output.json b/tests/data_inputs/redaction_order_config_output.json new file mode 100644 index 0000000..06be666 --- /dev/null +++ b/tests/data_inputs/redaction_order_config_output.json @@ -0,0 +1,704 @@ +[ + { + "incidence":8, + "population_value":10000, + "expected_to_redact":true, + "year":2020, + "month":1, + "zip_code":1, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":55, + "population_value":6000, + "expected_to_redact":true, + "year":2020, + "month":1, + "zip_code":3, + "is_anonymous":true, + "is_redacted":true, + "peer_group":"[{\"year\":2020,\"zip_code\":3}]", + "redacted_peers":"[{\"zip_code\":3}]", + "redaction_reason":"{\"year\":2020,\"month\":9,\"zip_code\":3} was a small cell" + }, + { + "incidence":56, + "population_value":1500, + "expected_to_redact":true, + "year":2020, + "month":1, + "zip_code":5, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":62, + "population_value":10000, + "expected_to_redact":true, + "year":2020, + "month":2, + "zip_code":1, + "is_anonymous":true, + "is_redacted":true, + "peer_group":"[{\"year\":2020,\"zip_code\":1}]", + "redacted_peers":"[{\"zip_code\":1}]", + "redaction_reason":"{\"year\":2020,\"month\":1,\"zip_code\":1} was a small cell" + }, + { + "incidence":7, + "population_value":4000, + "expected_to_redact":true, + "year":2020, + "month":2, + "zip_code":4, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":42, + "population_value":1500, + "expected_to_redact":true, + "year":2020, + "month":2, + "zip_code":5, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":8, + "population_value":7000, + "expected_to_redact":true, + "year":2020, + "month":3, + "zip_code":2, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":97, + "population_value":1500, + "expected_to_redact":true, + "year":2020, + "month":3, + "zip_code":5, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":10, + "population_value":4000, + "expected_to_redact":true, + "year":2020, + "month":4, + "zip_code":4, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":88, + "population_value":1500, + "expected_to_redact":true, + "year":2020, + "month":4, + "zip_code":5, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":22, + "population_value":4000, + "expected_to_redact":true, + "year":2020, + "month":5, + "zip_code":4, + "is_anonymous":true, + "is_redacted":true, + "peer_group":"[{\"year\":2020,\"month\":5}]", + "redacted_peers":"[{\"zip_code\":5},{\"zip_code\":4}]", + "redaction_reason":"{\"year\":2020,\"month\":5,\"zip_code\":5} was a small cell" + }, + { + "incidence":13, + "population_value":1500, + "expected_to_redact":true, + "year":2020, + "month":5, + "zip_code":5, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":91, + "population_value":4000, + "expected_to_redact":true, + "year":2020, + "month":6, + "zip_code":4, + "is_anonymous":true, + "is_redacted":true, + "peer_group":"[{\"year\":2020,\"month\":6}]", + "redacted_peers":"[{\"zip_code\":5},{\"zip_code\":4}]", + "redaction_reason":"{\"year\":2020,\"month\":6,\"zip_code\":5} was a small cell" + }, + { + "incidence":93, + "population_value":1500, + "expected_to_redact":true, + "year":2020, + "month":6, + "zip_code":5, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":6, + "population_value":7000, + "expected_to_redact":true, + "year":2020, + "month":7, + "zip_code":2, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":22, + "population_value":1500, + "expected_to_redact":true, + "year":2020, + "month":7, + "zip_code":5, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":54, + "population_value":4000, + "expected_to_redact":true, + "year":2020, + "month":8, + "zip_code":4, + "is_anonymous":true, + "is_redacted":true, + "peer_group":"[{\"year\":2020,\"month\":8}]", + "redacted_peers":"[{\"zip_code\":5},{\"zip_code\":4}]", + "redaction_reason":"{\"year\":2020,\"month\":8,\"zip_code\":5} was a small cell" + }, + { + "incidence":34, + "population_value":1500, + "expected_to_redact":true, + "year":2020, + "month":8, + "zip_code":5, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":6, + "population_value":6000, + "expected_to_redact":true, + "year":2020, + "month":9, + "zip_code":3, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":43, + "population_value":1500, + "expected_to_redact":true, + "year":2020, + "month":9, + "zip_code":5, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":69, + "population_value":4000, + "expected_to_redact":true, + "year":2020, + "month":10, + "zip_code":4, + "is_anonymous":true, + "is_redacted":true, + "peer_group":"[{\"year\":2020,\"month\":10}]", + "redacted_peers":"[{\"zip_code\":5},{\"zip_code\":4}]", + "redaction_reason":"{\"year\":2020,\"month\":10,\"zip_code\":5} was a small cell" + }, + { + "incidence":9, + "population_value":1500, + "expected_to_redact":true, + "year":2020, + "month":10, + "zip_code":5, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":13, + "population_value":4000, + "expected_to_redact":true, + "year":2020, + "month":11, + "zip_code":4, + "is_anonymous":true, + "is_redacted":true, + "peer_group":"[{\"year\":2020,\"month\":11}]", + "redacted_peers":"[{\"zip_code\":5},{\"zip_code\":4}]", + "redaction_reason":"{\"year\":2020,\"month\":11,\"zip_code\":5} was a small cell" + }, + { + "incidence":29, + "population_value":1500, + "expected_to_redact":true, + "year":2020, + "month":11, + "zip_code":5, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":77, + "population_value":4000, + "expected_to_redact":true, + "year":2020, + "month":12, + "zip_code":4, + "is_anonymous":true, + "is_redacted":true, + "peer_group":"[{\"year\":2020,\"month\":12}]", + "redacted_peers":"[{\"zip_code\":5},{\"zip_code\":4}]", + "redaction_reason":"{\"year\":2020,\"month\":12,\"zip_code\":5} was a small cell" + }, + { + "incidence":68, + "population_value":1500, + "expected_to_redact":true, + "year":2020, + "month":12, + "zip_code":5, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":64, + "population_value":10000, + "expected_to_redact":true, + "year":2021, + "month":1, + "zip_code":1, + "is_anonymous":true, + "is_redacted":true, + "peer_group":"[{\"month\":1,\"zip_code\":1}]", + "redacted_peers":"[{\"zip_code\":1}]", + "redaction_reason":"{\"year\":2020,\"month\":1,\"zip_code\":1} was a small cell" + }, + { + "incidence":12, + "population_value":4000, + "expected_to_redact":true, + "year":2021, + "month":1, + "zip_code":4, + "is_anonymous":true, + "is_redacted":true, + "peer_group":"[{\"year\":2021,\"month\":1}]", + "redacted_peers":"[{\"zip_code\":5},{\"zip_code\":4}]", + "redaction_reason":"{\"year\":2021,\"month\":1,\"zip_code\":5} was a small cell" + }, + { + "incidence":83, + "population_value":1500, + "expected_to_redact":true, + "year":2021, + "month":1, + "zip_code":5, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":40, + "population_value":4000, + "expected_to_redact":true, + "year":2021, + "month":2, + "zip_code":4, + "is_anonymous":true, + "is_redacted":true, + "peer_group":"[{\"year\":2021,\"month\":2}]", + "redacted_peers":"[{\"zip_code\":5},{\"zip_code\":4}]", + "redaction_reason":"{\"year\":2021,\"month\":2,\"zip_code\":5} was a small cell" + }, + { + "incidence":42, + "population_value":1500, + "expected_to_redact":true, + "year":2021, + "month":2, + "zip_code":5, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":24, + "population_value":7000, + "expected_to_redact":true, + "year":2021, + "month":3, + "zip_code":2, + "is_anonymous":true, + "is_redacted":true, + "peer_group":"[{\"month\":3,\"zip_code\":2}]", + "redacted_peers":"[{\"zip_code\":2}]", + "redaction_reason":"{\"year\":2020,\"month\":3,\"zip_code\":2} was a small cell" + }, + { + "incidence":60, + "population_value":4000, + "expected_to_redact":true, + "year":2021, + "month":3, + "zip_code":4, + "is_anonymous":true, + "is_redacted":true, + "peer_group":"[{\"year\":2021,\"month\":3}]", + "redacted_peers":"[{\"zip_code\":5},{\"zip_code\":4}]", + "redaction_reason":"{\"year\":2021,\"month\":3,\"zip_code\":5} was a small cell" + }, + { + "incidence":89, + "population_value":1500, + "expected_to_redact":true, + "year":2021, + "month":3, + "zip_code":5, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":59, + "population_value":4000, + "expected_to_redact":true, + "year":2021, + "month":4, + "zip_code":4, + "is_anonymous":true, + "is_redacted":true, + "peer_group":"[{\"year\":2021,\"month\":4}]", + "redacted_peers":"[{\"zip_code\":5},{\"zip_code\":4}]", + "redaction_reason":"{\"year\":2021,\"month\":4,\"zip_code\":5} was a small cell" + }, + { + "incidence":45, + "population_value":1500, + "expected_to_redact":true, + "year":2021, + "month":4, + "zip_code":5, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":26, + "population_value":4000, + "expected_to_redact":true, + "year":2021, + "month":5, + "zip_code":4, + "is_anonymous":true, + "is_redacted":true, + "peer_group":"[{\"year\":2021,\"month\":5}]", + "redacted_peers":"[{\"zip_code\":5},{\"zip_code\":4}]", + "redaction_reason":"{\"year\":2021,\"month\":5,\"zip_code\":5} was a small cell" + }, + { + "incidence":33, + "population_value":1500, + "expected_to_redact":true, + "year":2021, + "month":5, + "zip_code":5, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":66, + "population_value":4000, + "expected_to_redact":true, + "year":2021, + "month":6, + "zip_code":4, + "is_anonymous":true, + "is_redacted":true, + "peer_group":"[{\"year\":2021,\"month\":6}]", + "redacted_peers":"[{\"zip_code\":5},{\"zip_code\":4}]", + "redaction_reason":"{\"year\":2021,\"month\":6,\"zip_code\":5} was a small cell" + }, + { + "incidence":13, + "population_value":1500, + "expected_to_redact":true, + "year":2021, + "month":6, + "zip_code":5, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":85, + "population_value":7000, + "expected_to_redact":true, + "year":2021, + "month":7, + "zip_code":2, + "is_anonymous":true, + "is_redacted":true, + "peer_group":"[{\"month\":7,\"zip_code\":2}]", + "redacted_peers":"[{\"zip_code\":2}]", + "redaction_reason":"{\"year\":2020,\"month\":7,\"zip_code\":2} was a small cell" + }, + { + "incidence":86, + "population_value":4000, + "expected_to_redact":true, + "year":2021, + "month":7, + "zip_code":4, + "is_anonymous":true, + "is_redacted":true, + "peer_group":"[{\"year\":2021,\"month\":7}]", + "redacted_peers":"[{\"zip_code\":5},{\"zip_code\":4}]", + "redaction_reason":"{\"year\":2021,\"month\":7,\"zip_code\":5} was a small cell" + }, + { + "incidence":60, + "population_value":1500, + "expected_to_redact":true, + "year":2021, + "month":7, + "zip_code":5, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":40, + "population_value":4000, + "expected_to_redact":true, + "year":2021, + "month":8, + "zip_code":4, + "is_anonymous":true, + "is_redacted":true, + "peer_group":"[{\"year\":2021,\"month\":8}]", + "redacted_peers":"[{\"zip_code\":5},{\"zip_code\":4}]", + "redaction_reason":"{\"year\":2021,\"month\":8,\"zip_code\":5} was a small cell" + }, + { + "incidence":31, + "population_value":1500, + "expected_to_redact":true, + "year":2021, + "month":8, + "zip_code":5, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":86, + "population_value":6000, + "expected_to_redact":true, + "year":2021, + "month":9, + "zip_code":3, + "is_anonymous":true, + "is_redacted":true, + "peer_group":"[{\"month\":9,\"zip_code\":3}]", + "redacted_peers":"[{\"zip_code\":3}]", + "redaction_reason":"{\"year\":2020,\"month\":9,\"zip_code\":3} was a small cell" + }, + { + "incidence":19, + "population_value":4000, + "expected_to_redact":true, + "year":2021, + "month":9, + "zip_code":4, + "is_anonymous":true, + "is_redacted":true, + "peer_group":"[{\"year\":2021,\"month\":9}]", + "redacted_peers":"[{\"zip_code\":5},{\"zip_code\":4}]", + "redaction_reason":"{\"year\":2021,\"month\":9,\"zip_code\":5} was a small cell" + }, + { + "incidence":83, + "population_value":1500, + "expected_to_redact":true, + "year":2021, + "month":9, + "zip_code":5, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":87, + "population_value":4000, + "expected_to_redact":true, + "year":2021, + "month":10, + "zip_code":4, + "is_anonymous":true, + "is_redacted":true, + "peer_group":"[{\"year\":2021,\"month\":10}]", + "redacted_peers":"[{\"zip_code\":5},{\"zip_code\":4}]", + "redaction_reason":"{\"year\":2021,\"month\":10,\"zip_code\":5} was a small cell" + }, + { + "incidence":77, + "population_value":1500, + "expected_to_redact":true, + "year":2021, + "month":10, + "zip_code":5, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":11, + "population_value":4000, + "expected_to_redact":true, + "year":2021, + "month":11, + "zip_code":4, + "is_anonymous":true, + "is_redacted":true, + "peer_group":"[{\"year\":2021,\"month\":11}]", + "redacted_peers":"[{\"zip_code\":5},{\"zip_code\":4}]", + "redaction_reason":"{\"year\":2021,\"month\":11,\"zip_code\":5} was a small cell" + }, + { + "incidence":57, + "population_value":1500, + "expected_to_redact":true, + "year":2021, + "month":11, + "zip_code":5, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + }, + { + "incidence":98, + "population_value":4000, + "expected_to_redact":true, + "year":2021, + "month":12, + "zip_code":4, + "is_anonymous":true, + "is_redacted":true, + "peer_group":"[{\"year\":2021,\"month\":12}]", + "redacted_peers":"[{\"zip_code\":5},{\"zip_code\":4}]", + "redaction_reason":"{\"year\":2021,\"month\":12,\"zip_code\":5} was a small cell" + }, + { + "incidence":33, + "population_value":1500, + "expected_to_redact":true, + "year":2021, + "month":12, + "zip_code":5, + "is_anonymous":false, + "is_redacted":true, + "peer_group":null, + "redacted_peers":null, + "redaction_reason":"value meets redaction criteria \n'case\n when population_value < 2000 then true\n when incidence < 11 then true else false\nend\n'" + } +] \ No newline at end of file diff --git a/tests/data_inputs/redaction_order_data.csv b/tests/data_inputs/redaction_order_data.csv new file mode 100644 index 0000000..81a2fe5 --- /dev/null +++ b/tests/data_inputs/redaction_order_data.csv @@ -0,0 +1,121 @@ +zip_code,population_value,year,month,incidence,expected_to_redact +1,10000,2020,1,8,true +3,6000,2020,1,55,true +5,1500,2020,1,56,true +1,10000,2020,2,62,true +4,4000,2020,2,7,true +5,1500,2020,2,42,true +2,7000,2020,3,8,true +5,1500,2020,3,97,true +4,4000,2020,4,10,true +5,1500,2020,4,88,true +4,4000,2020,5,22,true +5,1500,2020,5,13,true +4,4000,2020,6,91,true +5,1500,2020,6,93,true +2,7000,2020,7,6,true +5,1500,2020,7,22,true +4,4000,2020,8,54,true +5,1500,2020,8,34,true +3,6000,2020,9,6,true +5,1500,2020,9,43,true +4,4000,2020,10,69,true +5,1500,2020,10,9,true +4,4000,2020,11,13,true +5,1500,2020,11,29,true +4,4000,2020,12,77,true +5,1500,2020,12,68,true +1,10000,2021,1,64,true +4,4000,2021,1,12,true +5,1500,2021,1,83,true +4,4000,2021,2,40,true +5,1500,2021,2,42,true +2,7000,2021,3,24,true +4,4000,2021,3,60,true +5,1500,2021,3,89,true +4,4000,2021,4,59,true +5,1500,2021,4,45,true +4,4000,2021,5,26,true +5,1500,2021,5,33,true +4,4000,2021,6,66,true +5,1500,2021,6,13,true +2,7000,2021,7,85,true +4,4000,2021,7,86,true +5,1500,2021,7,60,true +4,4000,2021,8,40,true +5,1500,2021,8,31,true +3,6000,2021,9,86,true +4,4000,2021,9,19,true +5,1500,2021,9,83,true +4,4000,2021,10,87,true +5,1500,2021,10,77,true +4,4000,2021,11,11,true +5,1500,2021,11,57,true +4,4000,2021,12,98,true +5,1500,2021,12,33,true +1,10000,2020,12,45,false +2,7000,2020,1,16,false +2,7000,2021,1,16,false +2,7000,2021,2,65,false +2,7000,2021,4,22,false +2,7000,2021,6,95,false +3,6000,2020,5,56,false +3,6000,2021,5,74,false +3,6000,2020,6,78,false +3,6000,2021,8,21,false +3,6000,2021,12,38,false +4,4000,2020,9,85,false +1,10000,2020,11,72,false +1,10000,2021,11,84,false +2,7000,2021,10,92,false +2,7000,2020,4,77,false +3,6000,2020,2,35,false +2,7000,2021,9,59,false +2,7000,2020,10,12,false +3,6000,2021,7,94,false +1,10000,2020,5,85,false +1,10000,2021,5,75,false +1,10000,2020,6,51,false +1,10000,2021,8,83,false +1,10000,2021,12,37,false +3,6000,2020,12,97,false +2,7000,2020,8,46,false +3,6000,2020,11,92,false +3,6000,2021,11,98,false +1,10000,2021,7,75,false +1,10000,2020,9,51,false +1,10000,2020,8,70,false +3,6000,2021,10,54,false +2,7000,2020,5,86,false +2,7000,2021,5,94,false +2,7000,2020,6,47,false +2,7000,2021,8,46,false +2,7000,2021,12,96,false +3,6000,2021,1,99,false +3,6000,2021,2,75,false +3,6000,2020,3,71,false +3,6000,2021,4,59,false +3,6000,2021,6,86,false +1,10000,2021,3,91,false +2,7000,2020,9,27,false +3,6000,2020,10,83,false +4,4000,2020,1,37,false +4,4000,2020,3,54,false +1,10000,2020,7,18,false +2,7000,2020,2,94,false +3,6000,2020,4,16,false +1,10000,2021,10,34,false +2,7000,2020,11,100,false +2,7000,2021,11,58,false +3,6000,2020,8,33,false +4,4000,2020,7,48,false +1,10000,2021,2,58,false +1,10000,2020,3,88,false +1,10000,2021,4,50,false +1,10000,2021,6,49,false +2,7000,2020,12,38,false +1,10000,2021,9,32,false +1,10000,2020,10,72,false +3,6000,2021,3,89,false +1,10000,2020,4,78,false +3,6000,2020,7,90,false diff --git a/tests/data_inputs/redaction_outputs.json b/tests/data_inputs/redaction_outputs.json index dbd20f6..8f54fa1 100644 --- a/tests/data_inputs/redaction_outputs.json +++ b/tests/data_inputs/redaction_outputs.json @@ -1,27 +1,4 @@ [ - { - "incidence": 11, - "population_value": 8000, - "purpose": "get suppressed due to peer (frankford within the same month for the same county)", - "expected_to_be_redacted": true, - "month": "1900-02-01", - "county": "county-sussex", - "municipality": "ellendale", - "zip_code": "zip-code-19941", - "is_anonymous": true, - "is_redacted": true, - "peer_group": { - "month": "1900-02-01", - "county": "county-sussex", - "municipality": "ellendale" - }, - "redacted_peers": { - "zip_code": [ - "zip-code-19941" - ] - }, - "redaction_reason": "value zip-code-18945 meets redaction criteria\n case\n when population_value is null then true\n when incidence = 0 then false\n when incidence < 11 and population_value >= 2500 and population_value < 20000 then true\n when population_value >= 20000 then false\n when population_value < 2500 then true\n else false\nend\n" - }, { "incidence": 200, "population_value": 2000, @@ -33,18 +10,9 @@ "zip_code": "zip-code-19967", "is_anonymous": false, "is_redacted": true, - "peer_group": { - "month": "1900-01-01", - "county": "county-sussex", - "municipality": "millville" - }, - "redacted_peers": { - "zip_code": [ - "zip-code-19967", - "zip-code-19970" - ] - }, - "redaction_reason": "value zip-code-19967 meets redaction criteria\n case\n when population_value is null then true\n when incidence = 0 then false\n when incidence < 11 and population_value >= 2500 and population_value < 20000 then true\n when population_value >= 20000 then false\n when population_value < 2500 then true\n else false\nend\n" + "peer_group": null, + "redacted_peers": null, + "redaction_reason": "value meets redaction criteria \n'case\n when population_value is null then true\n when incidence = 0 then false\n when incidence < 11 and population_value >= 2500 and population_value < 20000 then true\n when population_value >= 20000 then false\n when population_value < 2500 then true\n else false\nend\n'" }, { "incidence": 40, @@ -57,19 +25,37 @@ "zip_code": "zip-code-19970", "is_anonymous": true, "is_redacted": true, - "peer_group": { - "month": "1900-01-01", - "county": "county-sussex", - "municipality": "millville", - "zip_code": "zip-code-19967" - }, - "redacted_peers": { - "zip_code": [ - "zip-code-19967", - "zip-code-19970" - ] - }, - "redaction_reason": "value zip-code-19967 meets redaction criteria\n case\n when population_value is null then true\n when incidence = 0 then false\n when incidence < 11 and population_value >= 2500 and population_value < 20000 then true\n when population_value >= 20000 then false\n when population_value < 2500 then true\n else false\nend\n" + "peer_group": [ + { + "month": "1900-01-01", + "county": "county-sussex", + "municipality": "millville" + } + ], + "redacted_peers": [ + { + "zip_code": "zip-code-19967" + }, + { + "zip_code": "zip-code-19970" + } + ], + "redaction_reason": "{\"month\":\"1900-01-01\",\"county\":\"county-sussex\",\"municipality\":\"millville\",\"zip_code\":\"zip-code-19967\"} was a small cell" + }, + { + "incidence": 4, + "population_value": 300, + "purpose": "Provide a small cell that will accumulate latent suppression.", + "expected_to_be_redacted": true, + "month": "1900-01-01", + "county": "county-sussex", + "municipality": null, + "zip_code": "zip-code-19931", + "is_anonymous": false, + "is_redacted": true, + "peer_group": null, + "redacted_peers": null, + "redaction_reason": "value meets redaction criteria \n'case\n when population_value is null then true\n when incidence = 0 then false\n when incidence < 11 and population_value >= 2500 and population_value < 20000 then true\n when population_value >= 20000 then false\n when population_value < 2500 then true\n else false\nend\n'" }, { "incidence": 6, @@ -82,43 +68,64 @@ "zip_code": "zip-code-19945", "is_anonymous": false, "is_redacted": true, - "peer_group": { - "month": "1900-01-01", - "county": "county-sussex", - "municipality": null, - "zip_code": "zip-code-19931" - }, - "redacted_peers": { - "zip_code": [ - "zip-code-19931", - "zip-code-19945" - ] - }, - "redaction_reason": "values zip-code-19931, zip-code-19945 meet redaction criteria\n case\n when population_value is null then true\n when incidence = 0 then false\n when incidence < 11 and population_value >= 2500 and population_value < 20000 then true\n when population_value >= 20000 then false\n when population_value < 2500 then true\n else false\nend\n" + "peer_group": null, + "redacted_peers": null, + "redaction_reason": "value meets redaction criteria \n'case\n when population_value is null then true\n when incidence = 0 then false\n when incidence < 11 and population_value >= 2500 and population_value < 20000 then true\n when population_value >= 20000 then false\n when population_value < 2500 then true\n else false\nend\n'" }, { - "incidence": 4, - "population_value": 300, - "purpose": "Provide a small cell that will accumulate latent suppression.", + "incidence": 60, + "population_value": 8000, + "purpose": "This will get suppressed due to the small cell in the peer group.", "expected_to_be_redacted": true, "month": "1900-01-01", "county": "county-sussex", "municipality": null, - "zip_code": "zip-code-19931", - "is_anonymous": false, + "zip_code": "zip-code-20000", + "is_anonymous": true, + "is_redacted": true, + "peer_group": [ + { + "month": "1900-01-01", + "county": "county-sussex", + "municipality": null + } + ], + "redacted_peers": [ + { + "zip_code": "zip-code-19945" + }, + { + "zip_code": "zip-code-20000" + } + ], + "redaction_reason": "{\"month\":\"1900-01-01\",\"county\":\"county-sussex\",\"municipality\":null,\"zip_code\":\"zip-code-19945\"} was a small cell" + }, + { + "incidence": 11, + "population_value": 8000, + "purpose": "get suppressed due to peer (frankford within the same month for the same county)", + "expected_to_be_redacted": true, + "month": "1900-02-01", + "county": "county-sussex", + "municipality": "ellendale", + "zip_code": "zip-code-19941", + "is_anonymous": true, "is_redacted": true, - "peer_group": { - "month": "1900-01-01", - "county": "county-sussex", - "municipality": null - }, - "redacted_peers": { - "zip_code": [ - "zip-code-19931", - "zip-code-19945" - ] - }, - "redaction_reason": "values zip-code-19931, zip-code-19945 meet redaction criteria\n case\n when population_value is null then true\n when incidence = 0 then false\n when incidence < 11 and population_value >= 2500 and population_value < 20000 then true\n when population_value >= 20000 then false\n when population_value < 2500 then true\n else false\nend\n" + "peer_group": [ + { + "month": "1900-02-01", + "county": "county-sussex" + } + ], + "redacted_peers": [ + { + "zip_code": "zip-code-18945" + }, + { + "zip_code": "zip-code-19941" + } + ], + "redaction_reason": "{\"month\":\"1900-02-01\",\"county\":\"county-sussex\",\"municipality\":\"frankford\",\"zip_code\":\"zip-code-18945\"} was a small cell" }, { "incidence": 1, @@ -131,31 +138,8 @@ "zip_code": "zip-code-18945", "is_anonymous": false, "is_redacted": true, - "peer_group": { - "month": "1900-02-01", - "county": "county-sussex", - "municipality": "frankford" - }, - "redacted_peers": { - "zip_code": [ - "zip-code-18945" - ] - }, - "redaction_reason": "value zip-code-18945 meets redaction criteria\n case\n when population_value is null then true\n when incidence = 0 then false\n when incidence < 11 and population_value >= 2500 and population_value < 20000 then true\n when population_value >= 20000 then false\n when population_value < 2500 then true\n else false\nend\n" - }, - { - "incidence": 60, - "population_value": 8000, - "purpose": "This is a big cell in a largely suppressed peer group.", - "expected_to_be_redacted": false, - "month": "1900-01-01", - "county": "county-sussex", - "municipality": null, - "zip_code": "zip-code-20000", - "is_anonymous": true, - "is_redacted": false, "peer_group": null, "redacted_peers": null, - "redaction_reason": null + "redaction_reason": "value meets redaction criteria \n'case\n when population_value is null then true\n when incidence = 0 then false\n when incidence < 11 and population_value >= 2500 and population_value < 20000 then true\n when population_value >= 20000 then false\n when population_value < 2500 then true\n else false\nend\n'" } ] \ No newline at end of file diff --git a/tests/test_compound_threshold_expression.py b/tests/test_compound_threshold_expression.py index 9b51fa3..afd1874 100644 --- a/tests/test_compound_threshold_expression.py +++ b/tests/test_compound_threshold_expression.py @@ -157,15 +157,13 @@ def generate_population_redaction_rule_test_data(): population=1_000, expected_to_be_anonymous=False, ) - # This one will actually get latently redacted. - # TODO: The determinism on latent redaction, particularly across peers, needs improvement latent_low_incidence_big_population = PopulationTestRow( sex="M", zip="00001", incidence=2, population=20_000, expected_to_be_anonymous=True, - _expected_to_be_redacted=True, + _expected_to_be_redacted=False, ) dict_rows = [ row.to_dict() diff --git a/tests/test_initial_metrics.py b/tests/test_initial_metrics.py index 3efeed9..579eb63 100644 --- a/tests/test_initial_metrics.py +++ b/tests/test_initial_metrics.py @@ -173,5 +173,5 @@ def test_explicit_subsequent_metric(self, explicit_config): "county": "B", "zip_code": 23456, "is_anonymous": True, - "is_redacted": False, + "is_redacted": True, } in results diff --git a/tests/test_join_expressions.py b/tests/test_join_expressions.py index 000c2e6..c2958d5 100644 --- a/tests/test_join_expressions.py +++ b/tests/test_join_expressions.py @@ -92,11 +92,15 @@ def s3_files_prefix(mock_s3_bucket, mocked_s3_client): class TestJoinExpressions: @pytest.fixture() - def joined_config(self, mock_s3_bucket): + def joined_config(self, mock_s3_bucket, moto_server): return Config( datasource=DataSource( connection_type="s3", - parameters={"bucket": mock_s3_bucket, "key": "", **TEST_S3_PARAMS}, + parameters={ + "bucket": mock_s3_bucket, + "key": "", + **TEST_S3_PARAMS | {"endpoint": moto_server.replace("http://", "")}, + }, ), redaction_expression=REDACTION_EXPRESSION, datasets=[ @@ -122,12 +126,13 @@ def test_join_expression_redaction( joined_config, moto_server, ): + engine = Engine( config=joined_config, output_prefix=s3_files_prefix, output_bucket=mock_s3_bucket, ) - engine.connector.endpoint = moto_server.replace("http://", "") + incidence_file = engine.get_absolute_source_file("test_joins_incidence.csv") population_file = engine.get_absolute_source_file("test_joins_population.csv") engine.datasets[0].sql = engine.datasets[0].sql.format( diff --git a/tests/test_mark_redaction.py b/tests/test_mark_redaction.py index 6428495..03eb4c3 100644 --- a/tests/test_mark_redaction.py +++ b/tests/test_mark_redaction.py @@ -46,10 +46,22 @@ def test_basic_mark_redaction(file_system_config): engine.run() validation_sql = f""" select * from '{output_file}' """ t = engine.connector.duckdb_connection.sql(validation_sql) - anonymous_count, *_ = t.filter("not is_anonymous").count("*").fetchone() + non_anonymous_count, *_ = t.filter("not is_anonymous").count("*").fetchone() redaction_count, *_ = t.filter("is_redacted").count("*").fetchone() - assert anonymous_count == 1 - assert redaction_count == 2 + assert non_anonymous_count == 1 + assert redaction_count == 8 + + ###### Test first_order_only + file_system_config.datasets[0].suppression_strategies[ + 0 + ].parameters.first_order_only = True + engine = Engine(config=file_system_config) + output_file = "/tmp/output.parquet" + engine.run() + validation_sql = f""" select * from '{output_file}' """ + t = engine.connector.duckdb_connection.sql(validation_sql) + redaction_count, *_ = t.filter("is_redacted").count("*").fetchone() + assert redaction_count == 4 # confirm that suppression strategies are deserialized correctly (i.e., assert no error) file_system_config.datasets[0].suppression_strategies[0] = { diff --git a/tests/test_multi_dimensional_redaction.py b/tests/test_multi_dimensional_redaction.py new file mode 100644 index 0000000..4a82a94 --- /dev/null +++ b/tests/test_multi_dimensional_redaction.py @@ -0,0 +1,171 @@ +import os +import json +import yaml + +from engine import Engine +from config import ( + Config, + DatasetConfig, + DataSource, + Metric, + Aggregations, + MarkRedacted, + MarkRedactedParameters, +) + +from conftest import DATA_INPUTS_DIRECTORY, DATA_OUTPUTS_DIRECTORY + +multi_dimensional_redaction_file = os.path.join( + DATA_INPUTS_DIRECTORY, "multi-dimensional-revelation.json" +) + +redaction_order_config_file = os.path.join( + DATA_INPUTS_DIRECTORY, "redaction_order_config.yml" +) + +redaction_order_data_file = os.path.join( + DATA_INPUTS_DIRECTORY, "redaction_order_data.csv" +) + +redaction_order_output_file = os.path.join( + DATA_INPUTS_DIRECTORY, "redaction_order_config_output.json" +) + +multi_dimensional_redaction_config = Config( + redaction_expression="incidence < 11", + datasource=DataSource( + connection_type="file", + parameters={"output_directory": "/tmp/"}, + ), + datasets=[ + DatasetConfig( + dimensions=["county", "month"], + source_file=multi_dimensional_redaction_file, + metrics=[ + Metric( + aggregation=Aggregations.SUM, + column="incidence", + alias="incidence", + initial=True, + subsequent=True, + ), + Metric( + aggregation=Aggregations.ANY_VALUE, + column="expected_to_be_redacted", + alias="expected_to_be_redacted", + initial=True, + subsequent=True, + ), + Metric( + aggregation=Aggregations.ANY_VALUE, + column="note", + alias="note", + initial=True, + subsequent=True, + ), + ], + suppression_strategies=[ + MarkRedacted( + parameters=MarkRedactedParameters(redacted_dimension="county") + ) + ], + ) + ], +) + + +def test_multi_dimensional_redaction(): + engine = Engine(config=multi_dimensional_redaction_config) + engine.run() + output = engine.db.table(engine.datasets[0].output_file).order("county, month") + errors = ( + output.filter("expected_to_be_redacted != is_redacted") + .to_df() + .to_dict(orient="records") + ) + + assert ( + not errors + ), f"The following errors were found\n{json.dumps(errors, indent=2)}" + + +def test_multi_dimensional_redaction_with_redaction_order_dimensions(): + with open(redaction_order_config_file, "r") as f: + json_data = yaml.safe_load(f) + config = Config(**json_data) + config.datasets[0].source_file = redaction_order_data_file + engine = Engine(config=config) + db = engine.db + engine.run() + output = db.table(engine.datasets[0].output_file) + expected_to_redact = ( + output.filter("is_redacted") + .select("* replace(incidence::int as incidence)") + .order("year, month, zip_code, peer_group nulls first") + ) + expected_to_redact.to_df().to_json( + redaction_order_output_file, orient="records", indent=2 + ) + errors = ( + output.filter("expected_to_redact != is_redacted") + .to_df() + .to_dict(orient="records") + ) + assert ( + not errors + ), f"The following errors were found\n{json.dumps(errors, indent=2)}" + + # now, let's add a new small cell to see if it impacts the previous result. + new_data = db.sql( + """ + select zip_code, population_value, year, month, incidence, expected_to_redact + from output + union + select zip_code, population_value, 2022 as year, month, case when zip_code = '4' then 4 else incidence end as incidence, + zip_code in ('4', '5') as expected_to_redact + from output + where year = 2021 + and month = 1 + """ + ) + output_file = redaction_order_data_file.replace( + DATA_INPUTS_DIRECTORY, DATA_OUTPUTS_DIRECTORY + ) + new_data.to_csv(output_file) + config.datasets[0].source_file = output_file + engine = Engine(config=config) + engine.run() + output = engine.db.table(engine.datasets[0].output_file) + errors = ( + output.filter("expected_to_redact != is_redacted") + .to_df() + .to_dict(orient="records") + ) + assert ( + not errors + ), f"The following errors were found\n{json.dumps(errors, indent=2)}" + + +def generate_redaction_order_data(): + import duckdb + + db = duckdb.connect() + useful_data = [ + {"zip_code": 1, "population_value": 10000}, + {"zip_code": 2, "population_value": 7000}, + {"zip_code": 3, "population_value": 6000}, + {"zip_code": 4, "population_value": 4000}, + {"zip_code": 5, "population_value": 1500}, + ] + db.sql( + f"""\ + select zip_code, population_value, unnest([2020, 2021]) as year, month, + (random() * (100 - 5) )::int + 5 as incidence + from (select unnest({useful_data}, recursive := true)) as data, + generate_series(1, 12) as s(month) + """ + ).to_csv(redaction_order_data_file) + + +if __name__ == "__main__": + generate_redaction_order_data() diff --git a/tests/test_redaction_reasons.py b/tests/test_redaction_reasons.py index 7014418..1f30c36 100644 --- a/tests/test_redaction_reasons.py +++ b/tests/test_redaction_reasons.py @@ -108,7 +108,9 @@ def test_explicit_subsequent_metric(self, explicit_config): engine = Engine(config=explicit_config) engine.run() - output = engine.db.read_csv(engine.active_dataset.output_file) # noqa: F841 + output = engine.db.read_csv( # noqa: F841 + engine.active_dataset.output_file + ).order("month, county, municipality, zip_code") results = [ r[0] for r in engine.db.sql( diff --git a/tests/test_that_schema_is_updated.py b/tests/test_that_schema_is_updated.py new file mode 100644 index 0000000..b3c3294 --- /dev/null +++ b/tests/test_that_schema_is_updated.py @@ -0,0 +1,16 @@ +import os +from conftest import ROOT_DIR + + +def test_that_schema_is_updated(): + schema_file = os.path.join(ROOT_DIR, "schemas", "config_schema.json") + with open(schema_file, "r") as f: + data = f.read() + from schemas.generate import write_config_to_file + + write_config_to_file() + with open(schema_file, "r") as f: + new_data = f.read() + assert ( + data == new_data + ), "You changed the json schema for configs, please run `python schemas/generate.py` to update the schema."