Release 0.3.0 + 0.3.1 (validated via ICA full-pipeline runs)

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.2.18
+current_version = 0.3.1
 commit = True
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>[a-z0-9+]+)

.github/workflows/build.yaml

@@ -17,6 +17,22 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: false
+
+          # all of these default to true, but feel free to set to
+          # "false" if necessary for your workflow
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          docker-images: true
+          swap-storage: true
+
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v3
         with:
@@ -27,7 +43,8 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Build and push image
+      - name: Build image
+        id: build
         run: |
           if [[ "${{ matrix.build_name }}" == "base" ]]; then
             docker_tag=bolt:${GITHUB_REF_NAME#v};
@@ -37,10 +54,26 @@ jobs:
             dockerfile_fn=Dockerfile.${{ matrix.build_name }};
           fi;
 
-          # Build and push to GHCR umccr registry
           docker build \
             --platform linux/amd64 \
+            --load \
             -f docker/${dockerfile_fn} \
             -t ghcr.io/umccr/${docker_tag} \
-            --push \
             .;
+
+          echo "docker_tag=ghcr.io/umccr/${docker_tag}" >> $GITHUB_OUTPUT
+
+      - name: Smoke test
+        run: |
+          docker run --rm ${{ steps.build.outputs.docker_tag }} bolt --version
+
+          case "${{ matrix.build_name }}" in
+            multiqc) docker run --rm ${{ steps.build.outputs.docker_tag }} multiqc --version ;;
+            gpgr)    docker run --rm ${{ steps.build.outputs.docker_tag }} Rscript -e 'library(gpgr); cat("gpgr ok\n")' ;;
+            pcgr)    docker run --rm ${{ steps.build.outputs.docker_tag }} conda run --no-capture-output -n pcgr pcgr --version ;;
+            snpeff)  docker run --rm ${{ steps.build.outputs.docker_tag }} bash -c 'snpEff -version 2>&1 | head -1' ;;
+            circos)  docker run --rm ${{ steps.build.outputs.docker_tag }} circos --version ;;
+          esac
+
+      - name: Push image
+        run: docker push ${{ steps.build.outputs.docker_tag }}

.gitignore

@@ -5,3 +5,5 @@ __pycache__/
 build/
 venv/
 working/
+data/
+workspace/

CHANGELOG.md

@@ -1,6 +1,15 @@
 # bolt changelog
 
-## dev
+## 0.3.1
+
+- Fix `ModuleNotFoundError: No module named 'pkg_resources'` in `bolt:0.3.0-multiqc` — add `setuptools <81` to conda env
+- Fix `merge_vcf_files` producing wrong output filename — `Path.with_suffix()` was stripping `.pass` component; use explicit path concatenation instead
+- Fix VCF writers not closed in `transfer_annotations_somatic` and `transfer_annotations_germline` — BGZip output could be truncated
+- Fix `split_vcf` writing uncompressed plain `.vcf` chunks — now uses `.vcf.gz` with `wz` mode
+- Fix `PCGR_ACTIONABILITY_TIER` VCF header description — updated to match stored short-form values (`1`,`2`,`3`,`4`,`N`)
+- Add regression test for chunk file compression (`test_chunks_are_gzipped`)
+
+## 0.3.0
 
 - [28](https://github.com/umccr/bolt/pull/28) - gpgr version bump to 2.2.12 for cancer report hypermutated flag fix
 
@@ -10,4 +19,6 @@
 
 - [3](https://github.com/scwatts/bolt/pull/3) - Improve PCGR / CPSR argument handling
 
-- [6](https://github.com/umccr/bolt/pull/6) - Change oncoanalyser v2.0.0 uptade, with switch sv caller from GRIPSS to eSVee
+- [6](https://github.com/umccr/bolt/pull/6) - Change oncoanalyser v2.0.0 update, with switch sv caller from GRIPSS to eSVee
+
+- [9](https://github.com/umccr/bolt/pull/9) Add hypermutation sample handling

bolt/common/constants.py

@@ -4,7 +4,12 @@
 ######################################
 ## Variation selection (annotation) ##
 ######################################
-MAX_SOMATIC_VARIANTS = 500_000
+
+# Cap below PCGR's 500k limit. PCGR silently drops variants or skips HTML
+# generation above 500k, and its multi-allelic decomposition can inflate
+# variant count beyond what bolt outputs. 50k margin absorbs this safely.
+# See: docs/adr/001-max-somatic-variants-450k.md
+MAX_SOMATIC_VARIANTS = 450_000
 MAX_SOMATIC_VARIANTS_GNOMAD_FILTER = 0.01
 
 
@@ -35,12 +40,51 @@
     'pathogenic',
     'uncertain_significance',
 }
-PCGR_TIERS_RESCUE = {
-    'TIER_1',
-    'TIER_2',
+PCGR_ACTIONABILITY_TIER_RESCUE = {
+    '1',
+    '2',
 }
 
 
+################################
+## Hypermutated report filter ##
+################################
+# Values match short forms written by transfer_annotations_somatic() ('1'=TIER_1, ..., 'N'=NONCODING).
+# Order is lowest clinical priority first: NONCODING dropped before TIER_1.
+PCGR_TIERS_FILTERING = (
+    'N',
+    '4',
+    '3',
+    '2',
+    '1',
+)
+
+VEP_IMPACTS_FILTER = (
+    'intergenic',
+    'intronic',
+    'downstream',
+    'upstream',
+    'impacts_other',
+)
+
+GENOMIC_REGIONS_FILTERING = (
+    'difficult',
+    'none',
+    'giab_conf',
+)
+
+HOTSPOT_FIELDS_FILTERING = (
+    'SAGE_HOTSPOT',
+    'hotspot',
+    'PCGR_MUTATION_HOTSPOT',
+)
+
+RETAIN_FIELDS_FILTERING = (
+    'PANEL',
+    *HOTSPOT_FIELDS_FILTERING,
+)
+
+
 ##################################################
 ## VCF FILTER tags and FORMAT, INFO annotations ##
 ##################################################
@@ -61,6 +105,8 @@ class VcfFilter(enum.Enum):
     ENCODE = 'ENCODE'
     GNOMAD_COMMON = 'gnomAD_common'
 
+    PCGR_COUNT_LIMIT = 'PCGR_count_limit'
+
     @property
     def namespace(self):
         return 'FILTER'
@@ -77,16 +123,14 @@ class VcfInfo(enum.Enum):
     SAGE_NOVEL = 'SAGE_NOVEL'
     SAGE_RESCUE = 'SAGE_RESCUE'
 
-    PCGR_TIER = 'PCGR_TIER'
+    PCGR_ACTIONABILITY_TIER = 'PCGR_ACTIONABILITY_TIER'
     PCGR_CSQ = 'PCGR_CSQ'
     PCGR_MUTATION_HOTSPOT = 'PCGR_MUTATION_HOTSPOT'
-    PCGR_CLINVAR_CLNSIG = 'PCGR_CLINVAR_CLNSIG'
+    PCGR_CLINVAR_CLASSIFICATION = 'PCGR_CLINVAR_CLASSIFICATION'
     PCGR_COSMIC_COUNT = 'PCGR_COSMIC_COUNT'
     PCGR_TCGA_PANCANCER_COUNT = 'PCGR_TCGA_PANCANCER_COUNT'
     PCGR_ICGC_PCAWG_COUNT = 'PCGR_ICGC_PCAWG_COUNT'
 
-    CPSR_FINAL_CLASSIFICATION = 'CPSR_FINAL_CLASSIFICATION'
-    CPSR_PATHOGENICITY_SCORE = 'CPSR_PATHOGENICITY_SCORE'
     CPSR_CLINVAR_CLASSIFICATION = 'CPSR_CLINVAR_CLASSIFICATION'
     CPSR_CSQ = 'CPSR_CSQ'
 
@@ -112,7 +156,7 @@ class VcfInfo(enum.Enum):
 
     GNOMAD_AF = 'gnomAD_AF'
 
-    PCGR_TIER_RESCUE = 'PCGR_TIER_RESCUE'
+    PCGR_ACTIONABILITY_TIER_RESCUE = 'PCGR_ACTIONABILITY_TIER_RESCUE'
     SAGE_HOTSPOT_RESCUE = 'SAGE_HOTSPOT_RESCUE'
     CLINICAL_POTENTIAL_RESCUE = 'CLINICAL_POTENTIAL_RESCUE'
 
@@ -121,6 +165,8 @@ class VcfInfo(enum.Enum):
     RESCUED_FILTERS_EXISTING = 'RESCUED_FILTERS_EXISTING'
     RESCUED_FILTERS_PENDING = 'RESCUED_FILTERS_PENDING'
 
+    PANEL = 'PANEL'
+
     @property
     def namespace(self):
         return 'INFO'
@@ -187,6 +233,12 @@ def namespace(self):
         'Description': f'gnomAD AF >= {MAX_GNOMAD_AF}',
     },
 
+    VcfFilter.PCGR_COUNT_LIMIT: {
+        'Description': (
+            f'Manually filtered to {MAX_SOMATIC_VARIANTS} variants to stay below the '
+            f'PCGR hard limit of 500,000 variants'
+        ),
+    },
 
     # INFO
     VcfInfo.TUMOR_AF: {
@@ -226,39 +278,42 @@ def namespace(self):
         'Description': 'Variant rescued by a matching SAGE call',
     },
 
-    VcfInfo.PCGR_TIER: {
+    VcfInfo.PCGR_ACTIONABILITY_TIER: {
         'Number': '1',
         'Type': 'String',
         'Description': (
-            'Tier reported by PCGR with the following meaning: TIER_1: strong clinical '
-            'significance; TIER_2: potential clinical significance; TIER_3: uncertain clinical '
-            'significance; TIER_4: other coding variants; NONCODING: other non-coding variants'
+            'Tier reported by PCGR: 1: strong clinical significance; '
+            '2: potential clinical significance; 3: uncertain clinical significance; '
+            '4: other coding variants; N: other non-coding variants'
         ),
     },
     VcfInfo.PCGR_CSQ: {
         'Number': '.',
         'Type': 'String',
         'Description': (
-            'Consequence annotations from Ensembl VEP. Format: Allele|Consequence|IMPACT|SYMBOL|'
-            'Gene|Feature_type|Feature|BIOTYPE|EXON|INTRON|HGVSc|HGVSp|cDNA_position|'
-            'CDS_position|Protein_position|Amino_acids|Codons|Existing_variation|ALLELE_NUM|'
-            'DISTANCE|STRAND|FLAGS|PICK|VARIANT_CLASS|SYMBOL_SOURCE|HGNC_ID|CANONICAL|'
-            'MANE_SELECT|MANE_PLUS_CLINICAL|TSL|APPRIS|CCDS|ENSP|SWISSPROT|TREMBL|UNIPARC|'
-            'UNIPROT_ISOFORM|RefSeq|DOMAINS|HGVS_OFFSET|AF|AFR_AF|AMR_AF|EAS_AF|EUR_AF|SAS_AF|'
-            'gnomAD_AF|gnomAD_AFR_AF|gnomAD_AMR_AF|gnomAD_ASJ_AF|gnomAD_EAS_AF|gnomAD_FIN_AF|'
-            'gnomAD_NFE_AF|gnomAD_OTH_AF|gnomAD_SAS_AF|CLIN_SIG|SOMATIC|PHENO|CHECK_REF|'
-            'NearestExonJB'
+            'Consequence annotations from Ensembl VEP. Format: '
+            'Allele|Consequence|IMPACT|SYMBOL|Gene|Feature_type|Feature|BIOTYPE|EXON|INTRON|HGVSc|'
+            'HGVSp|cDNA_position|CDS_position|Protein_position|Amino_acids|Codons|Existing_variation|'
+            'ALLELE_NUM|DISTANCE|STRAND|FLAGS|PICK|VARIANT_CLASS|SYMBOL_SOURCE|HGNC_ID|CANONICAL|'
+            'MANE|MANE_SELECT|MANE_PLUS_CLINICAL|TSL|APPRIS|CCDS|ENSP|SWISSPROT|TREMBL|UNIPARC|'
+            'UNIPROT_ISOFORM|RefSeq|DOMAINS|HGVS_OFFSET|gnomADe_AF|gnomADe_AFR_AF|gnomADe_AMR_AF|'
+            'gnomADe_ASJ_AF|gnomADe_EAS_AF|gnomADe_FIN_AF|gnomADe_MID_AF|gnomADe_NFE_AF|'
+            'gnomADe_REMAINING_AF|gnomADe_SAS_AF|gnomADg_AF|gnomADg_AFR_AF|gnomADg_AMI_AF|'
+            'gnomADg_AMR_AF|gnomADg_ASJ_AF|gnomADg_EAS_AF|gnomADg_FIN_AF|gnomADg_MID_AF|'
+            'gnomADg_NFE_AF|gnomADg_REMAINING_AF|gnomADg_SAS_AF|CLIN_SIG|SOMATIC|PHENO|CHECK_REF|'
+            'MOTIF_NAME|MOTIF_POS|HIGH_INF_POS|MOTIF_SCORE_CHANGE|TRANSCRIPTION_FACTORS|NearestExonJB|'
+            'MaxEntScan_alt|MaxEntScan_diff|MaxEntScan_ref'
         ),
     },
     VcfInfo.PCGR_MUTATION_HOTSPOT: {
         'Number': '.',
         'Type': 'String',
-        'Description': 'Known cancer mutation hotspot, as found in cancerhotspots.org_v2, Gene|Codon|Q-value',
+        'Description': 'Known cancer mutation hotspot, as found in cancerhotspots.org. Format: GeneSymbol|Entrez_ID|CodonRefAA|Alt_AA|Q-value',
     },
-    VcfInfo.PCGR_CLINVAR_CLNSIG: {
+    VcfInfo.PCGR_CLINVAR_CLASSIFICATION: {
         'Number': '.',
         'Type': 'String',
-        'Description': 'ClinVar clinical significance',
+        'Description': 'ClinVar - Overall clinical significance of variant on a five-tiered scale',
     },
     VcfInfo.PCGR_COSMIC_COUNT: {
         'Number': '1',
@@ -270,44 +325,27 @@ def namespace(self):
         'Type': 'Integer',
         'Description': 'Raw variant count across all tumor types',
     },
+    VcfInfo.HMF_HOTSPOT: {
+        'Number': '0',
+        'Type': 'Flag',
+        'Description': 'calculated by flag of overlapping values in field HMF from annotations/hotspots/hotspots.hmf.vcf.gz',
+    },
     VcfInfo.PCGR_ICGC_PCAWG_COUNT: {
         'Number': '1',
         'Type': 'Integer',
         'Description': 'Count of ICGC PCAWG hits',
     },
 
-    VcfInfo.CPSR_FINAL_CLASSIFICATION: {
-        'Number': '1',
-        'Type': 'String',
-        'Description': (
-            'Final variant classification based on the combination of CLINVAR_CLASSIFICTION (for '
-            'ClinVar-classified variants), and CPSR_CLASSIFICATION (for novel variants)'
-        ),
-    },
-    VcfInfo.CPSR_PATHOGENICITY_SCORE: {
-        'Number': '1',
-        'Type': 'Float',
-        'Description': 'Aggregated CPSR pathogenicity score',
-    },
     VcfInfo.CPSR_CLINVAR_CLASSIFICATION: {
         'Number': '1',
         'Type': 'String',
-        'Description': 'Clinical significance of variant on a five-tiered scale',
+        'Description': 'ClinVar - Overall clinical significance of variant on a five-tiered scale',
     },
     VcfInfo.CPSR_CSQ: {
         'Number': '.',
         'Type': 'String',
         'Description': (
-            'Consequence annotations from Ensembl VEP. Format: Allele|Consequence|IMPACT|SYMBOL|'
-            'Gene|Feature_type|Feature|BIOTYPE|EXON|INTRON|HGVSc|HGVSp|cDNA_position|CDS_position|'
-            'Protein_position|Amino_acids|Codons|Existing_variation|ALLELE_NUM|DISTANCE|STRAND|'
-            'FLAGS|PICK|VARIANT_CLASS|SYMBOL_SOURCE|HGNC_ID|CANONICAL|MANE_SELECT|'
-            'MANE_PLUS_CLINICAL|APPRIS|CCDS|ENSP|SWISSPROT|TREMBL|UNIPARC|UNIPROT_ISOFORM|RefSeq|'
-            'DOMAINS|HGVS_OFFSET|AF|AFR_AF|AMR_AF|EAS_AF|EUR_AF|SAS_AF|gnomAD_AF|gnomAD_AFR_AF|'
-            'gnomAD_AMR_AF|gnomAD_ASJ_AF|gnomAD_EAS_AF|gnomAD_FIN_AF|gnomAD_NFE_AF|gnomAD_OTH_AF|'
-            'gnomAD_SAS_AF|CLIN_SIG|SOMATIC|PHENO|CHECK_REF|MOTIF_NAME|MOTIF_POS|HIGH_INF_POS|'
-            'MOTIF_SCORE_CHANGE|TRANSCRIPTION_FACTORS|NearestExonJB|LoF|LoF_filter|LoF_flags|'
-            'LoF_info'
+            'Consequence annotations from Ensembl VEP. Format: Allele|Consequence|IMPACT|SYMBOL|Gene|Feature_type|Feature|BIOTYPE|EXON|INTRON|HGVSc|HGVSp|cDNA_position|CDS_position|Protein_position|Amino_acids|Codons|Existing_variation|ALLELE_NUM|DISTANCE|STRAND|FLAGS|PICK|VARIANT_CLASS|SYMBOL_SOURCE|HGNC_ID|CANONICAL|MANE|MANE_SELECT|MANE_PLUS_CLINICAL|TSL|APPRIS|CCDS|ENSP|SWISSPROT|TREMBL|UNIPARC|UNIPROT_ISOFORM|RefSeq|DOMAINS|HGVS_OFFSET|gnomADe_AF|gnomADe_AFR_AF|gnomADe_AMR_AF|gnomADe_ASJ_AF|gnomADe_EAS_AF|gnomADe_FIN_AF|gnomADe_MID_AF|gnomADe_NFE_AF|gnomADe_REMAINING_AF|gnomADe_SAS_AF|gnomADg_AF|gnomADg_AFR_AF|gnomADg_AMI_AF|gnomADg_AMR_AF|gnomADg_ASJ_AF|gnomADg_EAS_AF|gnomADg_FIN_AF|gnomADg_MID_AF|gnomADg_NFE_AF|gnomADg_REMAINING_AF|gnomADg_SAS_AF|CLIN_SIG|SOMATIC|PHENO|CHECK_REF|MOTIF_NAME|MOTIF_POS|HIGH_INF_POS|MOTIF_SCORE_CHANGE|TRANSCRIPTION_FACTORS|NearestExonJB|MaxEntScan_alt|MaxEntScan_diff|MaxEntScan_ref'
         ),
     },
 
@@ -316,7 +354,7 @@ def namespace(self):
         'Type': 'Flag',
         'Description': '',
     },
-    VcfInfo.PCGR_TIER_RESCUE: {
+    VcfInfo.PCGR_ACTIONABILITY_TIER_RESCUE: {
         'Number': '0',
         'Type': 'Flag',
         'Description': '',
@@ -345,11 +383,17 @@ def namespace(self):
     },
 
     VcfInfo.RESCUED_FILTERS_PENDING: {
-        'Number': '1',
+        'Number': '.',
         'Type': 'String',
         'Description': 'Filters pending prior to variant rescue',
     },
 
+    VcfInfo.PANEL: {
+        'Number': '0',
+        'Type': 'Flag',
+        'Description': 'UMCCR somatic panel CDS (2,000 bp padding)',
+    },
+
 
     # FORMAT
     VcfFormat.SAGE_AD: {
@@ -368,13 +412,11 @@ def namespace(self):
         'Description': 'Approximate read depth (reads with MQ=255 or with bad mates are filtered)',
     },
     VcfFormat.SAGE_SB: {
-        'Number': '1',
+        'Number': '2',
         'Type': 'Float',
-        'Description': 'Strand bias - percentage of first-in-pair reads',
+        'Description': 'Fragment strand bias - percentage of forward-orientation fragments (ref,alt)',
     },
 }
-
-
 #####################
 ##      Other      ##
 #####################