diff --git a/src/igvfd/schemas/analysis_step.json b/src/igvfd/schemas/analysis_step.json index 7d07619ff1..1add17f9ef 100644 --- a/src/igvfd/schemas/analysis_step.json +++ b/src/igvfd/schemas/analysis_step.json @@ -42,7 +42,7 @@ }, "properties": { "schema_version": { - "default": "15" + "default": "16" }, "analysis_step_types": { "title": "Analysis Step Types", diff --git a/src/igvfd/schemas/changelogs/analysis_step.md b/src/igvfd/schemas/changelogs/analysis_step.md index 1353db17dc..ecb60e2dc7 100644 --- a/src/igvfd/schemas/changelogs/analysis_step.md +++ b/src/igvfd/schemas/changelogs/analysis_step.md @@ -1,5 +1,20 @@ ## Changelog for *`analysis_step.json`* +### Schema version 16 + +* Adjust `input_content_types` enum list to replace `global differential expression` with `global differential expression per element`. +* Adjust `output_content_types` enum list to replace `global differential expression` with `global differential expression per element`. +* Adjust `input_content_types` enum list to replace `local differential expression` with `global differential expression per element`. +* Adjust `output_content_types` enum list to replace `local differential expression` with `global differential expression per element`. +* Adjust `input_content_types` enum list to replace `cis differential expression quantifications per guide` with `local differential expression per guide`. +* Adjust `output_content_types` enum list to replace `cis differential expression quantifications per guide` with `local differential expression per guide`. +* Adjust `input_content_types` enum list to replace `trans differential expression quantifications per guide` with `global differential expression per guide`. +* Adjust `output_content_types` enum list to replace `trans differential expression quantifications per guide` with `global differential expression per guide`. +* Adjust `input_content_types` enum list to replace `cis differential expression quantifications per element` with `local differential expression per element`. +* Adjust `output_content_types` enum list to replace `cis differential expression quantifications per element` with `local differential expression per element`. +* Adjust `input_content_types` enum list to replace `trans differential expression quantifications per element` with `global differential expression per element`. +* Adjust `output_content_types` enum list to replace `trans differential expression quantifications per element` with `global differential expression per element`. + ### Minor changes since schema version 15 * Extend `analysis_step_type` enum list to include `quality metrics calculation`. diff --git a/src/igvfd/schemas/changelogs/tabular_file.md b/src/igvfd/schemas/changelogs/tabular_file.md index bcb58716b3..d37484fc66 100644 --- a/src/igvfd/schemas/changelogs/tabular_file.md +++ b/src/igvfd/schemas/changelogs/tabular_file.md @@ -1,5 +1,14 @@ ## Changelog for *`tabular_file.json`* +### Schema version 26 + +* Adjust `content_type` enum list to replace `global differential expression` with `global differential expression per element`. +* Adjust `content_type` enum list to replace `local differential expression` with `global differential expression per element`. +* Adjust `content_type` enum list to replace `cis differential expression quantifications per guide` with `local differential expression per guide`. +* Adjust `content_type` enum list to replace `trans differential expression quantifications per guide` with `global differential expression per guide`. +* Adjust `content_type` enum list to replace `cis differential expression quantifications per element` with `local differential expression per element`. +* Adjust `content_type` enum list to replace `trans differential expression quantifications per element` with `global differential expression per element`. + ### Minor changes since schema version 25 * Extend `catalog_collections` enum list to include `genomic_elements_phenotypes`. diff --git a/src/igvfd/schemas/tabular_file.json b/src/igvfd/schemas/tabular_file.json index 229f290946..f370dcbecc 100644 --- a/src/igvfd/schemas/tabular_file.json +++ b/src/igvfd/schemas/tabular_file.json @@ -159,8 +159,6 @@ "co-localization scores", "coding variant effects", "differential chromatin contact quantifications", - "cis differential expression quantifications per element", - "cis differential expression quantifications per guide", "differential element quantifications", "differential open reading frame quantifications", "differential gene expression quantifications", @@ -184,11 +182,13 @@ "gene programs", "gene program regulators", "gene universe", - "global differential expression", + "global differential expression per element", + "global differential expression per guide", "guide quantifications", "guide RNA sequences", "guide RNA sequences reference", - "local differential expression", + "local differential expression per element", + "local differential expression per guide", "loci", "methylation counts", "minus strand modification state", @@ -219,8 +219,6 @@ "reporter variants", "spliceQTL", "target transcripts", - "trans differential expression quantifications per element", - "trans differential expression quantifications per guide", "transcript quantifications", "variant binding effects", "variant effects", @@ -319,7 +317,7 @@ "type": "object", "properties": { "schema_version": { - "default": "25" + "default": "26" }, "catalog_adapters": { "title": "IGVF Catalog Adapters", @@ -362,8 +360,6 @@ "cell annotations", "cell hashing barcodes", "chromatin contact quantifications", - "cis differential expression quantifications per element", - "cis differential expression quantifications per guide", "cis-trans regulatory differences", "co-localization scores", "coding variant effects", @@ -395,13 +391,15 @@ "gene program regulators", "gene quantifications", "gene universe", - "global differential expression", + "global differential expression per element", + "global differential expression per guide", "guide quantifications", "guide RNA sequences", "heritability enrichment", "index plate", "individual cells profile", - "local differential expression", + "local differential expression per element", + "local differential expression per guide", "loci", "machine learning model features", "marker genes", @@ -452,8 +450,6 @@ "spatial transcriptomics slide-level transcripts", "spliceQTL", "target transcripts", - "trans differential expression quantifications per element", - "trans differential expression quantifications per guide", "tissue positions", "transcript quantifications", "variants", @@ -486,8 +482,6 @@ "cell annotations": "A file containing cell classifications and/or labels based on their unique transcriptome and genomic signatures, usually generated from single cell sequencing experiments.", "cell hashing barcodes": "The barcodes used for cell hashing.", "chromatin contact quantifications": "A file containing chromatin conformation contacts.", - "cis differential expression quantifications per element": "Element-level differential expression results aggregated across guides targeting each element for nearby genes within a configured interval.", - "cis differential expression quantifications per guide": "Guide-level differential expression results for nearby genes within a configured interval around the guide target.", "cis-trans regulatory differences": "Classification by gene for type of cis- or trans-acting regulation.", "co-localization scores": "eQTL-GWAS co-localization p-values", "coding variant effects": "A file predicting effects of amino acid changes at each position of a given protein.", @@ -515,13 +509,15 @@ "gene program regulators": "Predicted regulators of a gene program.", "gene quantifications": "The quantified levels of genes.", "gene universe": "List of gene Ensembl ID and symbol using official IGVF GENCODE version (V43), used for downstream analysis of gene networks by the Cellular Program Networks WG.", - "global differential expression": "A file listing differentially expressed genes across the entire dataset.", + "global differential expression per element": "Element-level differential expression results aggregated across guides targeting each element across the full gene annotation.", + "global differential expression per guide": "Guide-level differential expression results across the full gene annotation.", "guide quantifications": "Information and sequencing read counts for each gRNA in a pooled CRISPR screen experiment.", "guide RNA sequences": "The sequences of guide RNAs designed for genome editing using CRISPR. Formatting should follow https://github.com/IGVF-DACC/checkfiles/blob/dev/src/schemas/table_schemas/guide_rna_sequences.json.", "heritability enrichment": "A file with enrichment scores identifying if genomic annotations contribute disproportionately to the heritability of a trait.", "index plate": "Describes the plate layout of sample index combinations (e.g., i7/i5) used for multiplexed sequencing libraries.", "individual cells profile": "CSV file with morphological feature measurements for each imaged cell.", - "local differential expression": "A file listing differentially expressed genes within a defined genomic region surrounding a perturbation.", + "local differential expression per element": "Element-level differential expression results aggregated across guides targeting each element for nearby genes within a configured interval.", + "local differential expression per guide": "Guide-level differential expression results for nearby genes within a configured interval around the guide target.", "loci": "A table listing genomic loci.", "external source data": "File containing a list of urls used as a source.", "exon constraint": "A file containing model estimates for the probability that a variant observed in the population is missense within an exon.", @@ -578,8 +574,6 @@ "target transcripts": "A GTF file containing annotations of targeted transcripts by the TAP-seq primer panel.", "tissue positions": "The spatial coordinates or positions of tissue samples or sections in experiments.", "transcript quantifications": "The quantified levels of transcripts.", - "trans differential expression quantifications per guide": "Guide-level differential expression results across the full gene annotation.", - "trans differential expression quantifications per element": "Element-level differential expression results aggregated across guides targeting each element across the full gene annotation.", "variants": "A table listing genetic variants, which may be called from IGVF data or collected from external resources.", "variant binding effects": "The genetic variant effects on transcription factor binding affinity.", "variant effects": "The genetic variant effects on gene expression.", diff --git a/src/igvfd/tests/data/inserts/analysis_step.json b/src/igvfd/tests/data/inserts/analysis_step.json index 1d8adeec87..3daa849589 100644 --- a/src/igvfd/tests/data/inserts/analysis_step.json +++ b/src/igvfd/tests/data/inserts/analysis_step.json @@ -153,7 +153,7 @@ ], "output_content_types": [ "variant effects", - "global differential expression" + "global differential expression per element" ] }, { @@ -218,7 +218,7 @@ "reads", "index", "variant to cluster mapping", - "global differential expression" + "global differential expression per element" ], "output_content_types": [ "fragments", @@ -280,10 +280,10 @@ "elements reference" ], "output_content_types": [ - "cis differential expression quantifications per guide", - "trans differential expression quantifications per guide", - "cis differential expression quantifications per element", - "trans differential expression quantifications per element", + "local differential expression per guide", + "global differential expression per guide", + "local differential expression per element", + "global differential expression per element", "pipeline dashboard" ] } diff --git a/src/igvfd/tests/data/inserts/tabular_file.json b/src/igvfd/tests/data/inserts/tabular_file.json index 8ae6a513e9..5128c6f5ce 100644 --- a/src/igvfd/tests/data/inserts/tabular_file.json +++ b/src/igvfd/tests/data/inserts/tabular_file.json @@ -1290,7 +1290,7 @@ ], "md5sum": "85d68e982987441946768df28b8e9af4", "file_format": "tsv", - "content_type": "global differential expression", + "content_type": "global differential expression per element", "reference_files": [ "IGVFFI0001GNRF" ], @@ -1891,17 +1891,17 @@ "lab": "danwei-huangfu", "award": "1U01HG012051-01", "aliases": [ - "igvf:tabular_file_cis_differential_expression_per_guide" + "igvf:tabular_file_local_differential_expression_per_guide" ], "status": "released", "release_timestamp": "2026-05-27T12:34:56Z", "md5sum": "4f4f16496f0f4ee0df909bbf0d0c4c69", "file_format": "tsv", - "content_type": "cis differential expression quantifications per guide", + "content_type": "local differential expression per guide", "reference_files": [ "IGVFFI0001GNRF" ], - "submitted_file_name": "/Users/igvf/igvf_files/cis_differential_expression_quantifications_per_guide.tsv.gz", + "submitted_file_name": "/Users/igvf/igvf_files/local_differential_expression_per_guide.tsv.gz", "file_size": 5310, "upload_status": "validated", "filtered": false, @@ -1914,17 +1914,17 @@ "lab": "danwei-huangfu", "award": "1U01HG012051-01", "aliases": [ - "igvf:tabular_file_trans_differential_expression_per_guide" + "igvf:tabular_file_global_differential_expression_per_guide" ], "status": "released", "release_timestamp": "2026-05-27T12:34:56Z", "md5sum": "5a5f16496f0f4ee0df909bbf0d0c4c70", "file_format": "tsv", - "content_type": "trans differential expression quantifications per guide", + "content_type": "global differential expression per guide", "reference_files": [ "IGVFFI0001GNRF" ], - "submitted_file_name": "/Users/igvf/igvf_files/trans_differential_expression_quantifications_per_guide.tsv.gz", + "submitted_file_name": "/Users/igvf/igvf_files/global_differential_expression_per_guide.tsv.gz", "file_size": 6520, "upload_status": "validated", "filtered": false, @@ -1937,17 +1937,17 @@ "lab": "danwei-huangfu", "award": "1U01HG012051-01", "aliases": [ - "igvf:tabular_file_cis_differential_expression_per_element" + "igvf:tabular_file_local_differential_expression_per_element" ], "status": "released", "release_timestamp": "2026-05-27T12:34:56Z", "md5sum": "6b6f16496f0f4ee0df909bbf0d0c4c71", "file_format": "tsv", - "content_type": "cis differential expression quantifications per element", + "content_type": "local differential expression per element", "reference_files": [ "IGVFFI0001GNRF" ], - "submitted_file_name": "/Users/igvf/igvf_files/cis_differential_expression_quantifications_per_element.tsv.gz", + "submitted_file_name": "/Users/igvf/igvf_files/local_differential_expression_per_element.tsv.gz", "file_size": 7180, "upload_status": "validated", "filtered": false, @@ -1960,17 +1960,17 @@ "lab": "danwei-huangfu", "award": "1U01HG012051-01", "aliases": [ - "igvf:tabular_file_trans_differential_expression_per_element" + "igvf:tabular_file_global_differential_expression_per_element" ], "status": "released", "release_timestamp": "2026-05-27T12:34:56Z", "md5sum": "7c7f16496f0f4ee0df909bbf0d0c4c72", "file_format": "tsv", - "content_type": "trans differential expression quantifications per element", + "content_type": "global differential expression per element", "reference_files": [ "IGVFFI0001GNRF" ], - "submitted_file_name": "/Users/igvf/igvf_files/trans_differential_expression_quantifications_per_element.tsv.gz", + "submitted_file_name": "/Users/igvf/igvf_files/global_differential_expression_per_element.tsv.gz", "file_size": 7395, "upload_status": "validated", "filtered": false, diff --git a/src/igvfd/tests/fixtures/schemas/analysis_step.py b/src/igvfd/tests/fixtures/schemas/analysis_step.py index 4dced97f9a..a28dbf0ebc 100644 --- a/src/igvfd/tests/fixtures/schemas/analysis_step.py +++ b/src/igvfd/tests/fixtures/schemas/analysis_step.py @@ -187,3 +187,30 @@ def analysis_step_v14(testapp, base_workflow): 'analysis_step_types': ['quantification'] } return item + + +@pytest.fixture +def analysis_step_v15(testapp, base_workflow): + item = { + 'schema_version': '15', + 'step_label': 'differential-expression-analysis-step', + 'title': 'Differential Expression Analysis Step', + 'input_content_types': [ + 'global differential expression', + 'local differential expression', + 'cis differential expression quantifications per guide', + 'trans differential expression quantifications per guide', + 'cis differential expression quantifications per element', + 'trans differential expression quantifications per element', + ], + 'output_content_types': [ + 'global differential expression', + 'local differential expression', + 'cis differential expression quantifications per guide', + 'trans differential expression quantifications per guide', + 'cis differential expression quantifications per element', + 'trans differential expression quantifications per element', + ], + 'analysis_step_types': ['differential expression analysis'] + } + return item diff --git a/src/igvfd/tests/fixtures/schemas/tabular_file.py b/src/igvfd/tests/fixtures/schemas/tabular_file.py index b61f8af5ec..44d56a218d 100644 --- a/src/igvfd/tests/fixtures/schemas/tabular_file.py +++ b/src/igvfd/tests/fixtures/schemas/tabular_file.py @@ -431,3 +431,63 @@ def tabular_file_v24(tabular_file): 'content_type': 'fold change over control' }) return item + + +@pytest.fixture +def tabular_file_v25a(tabular_file): + item = tabular_file.copy() + item.update({ + 'schema_version': '25', + 'content_type': 'global differential expression' + }) + return item + + +@pytest.fixture +def tabular_file_v25b(tabular_file): + item = tabular_file.copy() + item.update({ + 'schema_version': '25', + 'content_type': 'local differential expression' + }) + return item + + +@pytest.fixture +def tabular_file_v25c(tabular_file): + item = tabular_file.copy() + item.update({ + 'schema_version': '25', + 'content_type': 'cis differential expression quantifications per guide' + }) + return item + + +@pytest.fixture +def tabular_file_v25d(tabular_file): + item = tabular_file.copy() + item.update({ + 'schema_version': '25', + 'content_type': 'trans differential expression quantifications per guide' + }) + return item + + +@pytest.fixture +def tabular_file_v25e(tabular_file): + item = tabular_file.copy() + item.update({ + 'schema_version': '25', + 'content_type': 'cis differential expression quantifications per element' + }) + return item + + +@pytest.fixture +def tabular_file_v25f(tabular_file): + item = tabular_file.copy() + item.update({ + 'schema_version': '25', + 'content_type': 'trans differential expression quantifications per element' + }) + return item diff --git a/src/igvfd/tests/test_upgrade_analysis_step.py b/src/igvfd/tests/test_upgrade_analysis_step.py index 1b21722da2..5b532c3ee9 100644 --- a/src/igvfd/tests/test_upgrade_analysis_step.py +++ b/src/igvfd/tests/test_upgrade_analysis_step.py @@ -117,3 +117,40 @@ def test_analysis_step_upgrade_14_15(upgrader, analysis_step_v14): 'This analysis step\'s input_content_types included fold change over control, ' 'but has been upgraded to allelic effects.' ) in value['notes'] + + +def test_analysis_step_upgrade_15_16(upgrader, analysis_step_v15): + value = upgrader.upgrade('analysis_step', analysis_step_v15, current_version='15', target_version='16') + assert value['schema_version'] == '16' + expected = { + 'global differential expression per element', + 'local differential expression per guide', + 'global differential expression per guide', + 'local differential expression per element', + } + assert set(value['input_content_types']) == expected + assert set(value['output_content_types']) == expected + assert ( + 'This analysis step\'s input_content_types included global differential expression, ' + 'but has been upgraded to global differential expression per element.' + ) in value['notes'] + assert ( + 'This analysis step\'s input_content_types included local differential expression, ' + 'but has been upgraded to global differential expression per element.' + ) in value['notes'] + assert ( + 'This analysis step\'s input_content_types included cis differential expression quantifications per guide, ' + 'but has been upgraded to local differential expression per guide.' + ) in value['notes'] + assert ( + 'This analysis step\'s input_content_types included trans differential expression quantifications per guide, ' + 'but has been upgraded to global differential expression per guide.' + ) in value['notes'] + assert ( + 'This analysis step\'s input_content_types included cis differential expression quantifications per element, ' + 'but has been upgraded to local differential expression per element.' + ) in value['notes'] + assert ( + 'This analysis step\'s input_content_types included trans differential expression quantifications per element, ' + 'but has been upgraded to global differential expression per element.' + ) in value['notes'] diff --git a/src/igvfd/tests/test_upgrade_tabular_file.py b/src/igvfd/tests/test_upgrade_tabular_file.py index 891e496465..2433aadf46 100644 --- a/src/igvfd/tests/test_upgrade_tabular_file.py +++ b/src/igvfd/tests/test_upgrade_tabular_file.py @@ -163,3 +163,36 @@ def test_tabular_file_upgrade_24_25(upgrader, tabular_file_v24): assert value['schema_version'] == '25' assert 'This file\'s content_type was fold change over control, and changed to allelic effects via upgrade.' in value[ 'notes'] + + +def test_tabular_file_upgrade_25_26(upgrader, tabular_file_v25a, tabular_file_v25b, tabular_file_v25c, tabular_file_v25d, tabular_file_v25e, tabular_file_v25f): + value = upgrader.upgrade('tabular_file', tabular_file_v25a, current_version='25', target_version='26') + assert value['content_type'] == 'global differential expression per element' + assert value['schema_version'] == '26' + assert 'This file\'s content_type was global differential expression, but has been upgraded to global differential expression per element.' in value[ + 'notes'] + value = upgrader.upgrade('tabular_file', tabular_file_v25b, current_version='25', target_version='26') + assert value['content_type'] == 'global differential expression per element' + assert value['schema_version'] == '26' + assert 'This file\'s content_type was local differential expression, but has been upgraded to global differential expression per element.' in value[ + 'notes'] + value = upgrader.upgrade('tabular_file', tabular_file_v25c, current_version='25', target_version='26') + assert value['content_type'] == 'local differential expression per guide' + assert value['schema_version'] == '26' + assert 'This file\'s content_type was cis differential expression quantifications per guide, but has been upgraded to local differential expression per guide.' in value[ + 'notes'] + value = upgrader.upgrade('tabular_file', tabular_file_v25d, current_version='25', target_version='26') + assert value['content_type'] == 'global differential expression per guide' + assert value['schema_version'] == '26' + assert 'This file\'s content_type was trans differential expression quantifications per guide, but has been upgraded to global differential expression per guide.' in value[ + 'notes'] + value = upgrader.upgrade('tabular_file', tabular_file_v25e, current_version='25', target_version='26') + assert value['content_type'] == 'local differential expression per element' + assert value['schema_version'] == '26' + assert 'This file\'s content_type was cis differential expression quantifications per element, but has been upgraded to local differential expression per element.' in value[ + 'notes'] + value = upgrader.upgrade('tabular_file', tabular_file_v25f, current_version='25', target_version='26') + assert value['content_type'] == 'global differential expression per element' + assert value['schema_version'] == '26' + assert 'This file\'s content_type was trans differential expression quantifications per element, but has been upgraded to global differential expression per element.' in value[ + 'notes'] diff --git a/src/igvfd/upgrade/analysis_step.py b/src/igvfd/upgrade/analysis_step.py index 72b9fe80f4..32e4a40615 100644 --- a/src/igvfd/upgrade/analysis_step.py +++ b/src/igvfd/upgrade/analysis_step.py @@ -328,3 +328,31 @@ def analysis_step_14_15(value, system): value[key] = sorted(set(new_content_types)) if notes.strip() != '': value['notes'] = notes.strip() + + +@upgrade_step('analysis_step', '15', '16') +def analysis_step_15_16(value, system): + # https://igvf.atlassian.net/browse/IGVF-3530 + notes = value.get('notes', '') + upgrade_map = { + 'global differential expression': 'global differential expression per element', + 'local differential expression': 'global differential expression per element', + 'cis differential expression quantifications per guide': 'local differential expression per guide', + 'trans differential expression quantifications per guide': 'global differential expression per guide', + 'cis differential expression quantifications per element': 'local differential expression per element', + 'trans differential expression quantifications per element': 'global differential expression per element', + } + for key in ['input_content_types', 'output_content_types']: + new_content_types = [] + for content_type in value.get(key, []): + if content_type in upgrade_map: + new_content_types.append(upgrade_map[content_type]) + notes += ( + f' This analysis step\'s {key} included {content_type}, ' + f'but has been upgraded to {upgrade_map[content_type]}.' + ) + else: + new_content_types.append(content_type) + value[key] = sorted(set(new_content_types)) + if notes.strip() != '': + value['notes'] = notes.strip() diff --git a/src/igvfd/upgrade/file.py b/src/igvfd/upgrade/file.py index 723b2772e2..24f0126257 100644 --- a/src/igvfd/upgrade/file.py +++ b/src/igvfd/upgrade/file.py @@ -941,3 +941,26 @@ def signal_file_14_15(value, system): ) if notes.strip() != '': value['notes'] = notes.strip() + + +@upgrade_step('tabular_file', '25', '26') +def tabular_file_25_26(value, system): + # https://igvf.atlassian.net/browse/IGVF-3530 + notes = value.get('notes', '') + upgrade_map = { + 'global differential expression': 'global differential expression per element', + 'local differential expression': 'global differential expression per element', + 'cis differential expression quantifications per guide': 'local differential expression per guide', + 'trans differential expression quantifications per guide': 'global differential expression per guide', + 'cis differential expression quantifications per element': 'local differential expression per element', + 'trans differential expression quantifications per element': 'global differential expression per element', + } + old_content_type = value.get('content_type', '') + if old_content_type in upgrade_map: + value['content_type'] = upgrade_map[old_content_type] + notes += ( + f' This file\'s content_type was {old_content_type}, ' + f'but has been upgraded to {upgrade_map[old_content_type]}.' + ) + if notes.strip() != '': + value['notes'] = notes.strip()