From d7ca46b2ddb8aef1b1e9568d034b8fb66e0634d7 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen <37511834+qclayssen@users.noreply.github.com> Date: Wed, 27 Aug 2025 14:40:51 +1000 Subject: [PATCH 01/89] Add option for sash sigrap individual process input to cancer report (#11) * add params input for sigrap tools --- bolt/workflows/other/cancer_report.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bolt/workflows/other/cancer_report.py b/bolt/workflows/other/cancer_report.py index 983f596..60b4da3 100644 --- a/bolt/workflows/other/cancer_report.py +++ b/bolt/workflows/other/cancer_report.py @@ -28,6 +28,10 @@ @click.option('--purple_dir', required=True, type=click.Path(exists=True)) @click.option('--virusbreakend_dir', required=True, type=click.Path(exists=True)) +@click.option('--mutpat_dir', required=True, type=click.Path(exists=True)) + +@click.option('--hrdetect_file', required=True, type=click.Path(exists=True)) +@click.option('--chord_file', required=True, type=click.Path(exists=True)) @click.option('--dragen_hrd_fp', required=False, type=click.Path(exists=True)) @@ -99,6 +103,10 @@ def entry(ctx, **kwargs): --key_genes {kwargs['cancer_genes_fp']} \ --oncokb_genes {kwargs['oncokb_genes_fp']} \ \ + --mutpat_dir {kwargs['mutpat_dir']} \ + --hrdetect_file {kwargs['hrdetect_file']} \ + --chord_file {kwargs['chord_file']} \ + \ --img_dir {output_image_dir}/ \ --result_outdir {output_table_dir}/ \ --out_file {output_dir}/{kwargs['tumor_name']}.cancer_report.html From f33c48c7a96ebe4e2edb6cc0f00ea9fe01bdb7c9 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen <37511834+qclayssen@users.noreply.github.com> Date: Fri, 29 Aug 2025 15:47:40 +1000 Subject: [PATCH 02/89] Hypermutation Handling in PCGR (#9) --- .gitignore | 2 + CHANGELOG.md | 4 +- bolt/common/constants.py | 52 ++++- bolt/common/pcgr.py | 234 ++++++++++++++++++++-- bolt/logging_config.py | 34 ++++ bolt/util.py | 256 +++++++++++++++++++++--- bolt/workflows/other/cancer_report.py | 6 + bolt/workflows/smlv_germline/report.py | 4 + bolt/workflows/smlv_somatic/annotate.py | 59 +++--- bolt/workflows/smlv_somatic/filter.py | 3 + bolt/workflows/smlv_somatic/report.py | 85 +++++++- bolt/workflows/smlv_somatic/rescue.py | 7 + bolt/workflows/sv_somatic/annotate.py | 3 + conda/env/bolt_env.yml | 1 + pyproject.toml | 1 + 15 files changed, 677 insertions(+), 74 deletions(-) create mode 100644 bolt/logging_config.py diff --git a/.gitignore b/.gitignore index 59a0c10..e4ef844 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ __pycache__/ build/ venv/ working/ +data/ +workspace/ \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 88a28d9..90b7815 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,4 +8,6 @@ - [3](https://github.com/scwatts/bolt/pull/3) - Improve PCGR / CPSR argument handling -- [6](https://github.com/umccr/bolt/pull/6) - Change oncoanalyser v2.0.0 uptade, with switch sv caller from GRIPSS to eSVee \ No newline at end of file +- [6](https://github.com/umccr/bolt/pull/6) - Change oncoanalyser v2.0.0 uptade, with switch sv caller from GRIPSS to eSVee + +-[9](https://github.com/umccr/bolt/pull/9) Add hypermutation sample handling \ No newline at end of file diff --git a/bolt/common/constants.py b/bolt/common/constants.py index e38ed3d..29216bc 100644 --- a/bolt/common/constants.py +++ b/bolt/common/constants.py @@ -4,7 +4,7 @@ ###################################### ## Variation selection (annotation) ## ###################################### -MAX_SOMATIC_VARIANTS = 500_000 +MAX_SOMATIC_VARIANTS = 450_000 MAX_SOMATIC_VARIANTS_GNOMAD_FILTER = 0.01 @@ -41,6 +41,43 @@ } +################################ +## Hypermutated report filter ## +################################ +PCGR_TIERS_FILTERING = ( + 'TIER_1', + 'TIER_2', + 'TIER_3', + 'TIER_4', + 'NONCODING', +) + +VEP_IMPACTS_FILTER = ( + 'intergenic', + 'intronic', + 'downstream', + 'upstream', + 'impacts_other', +) + +GENOMIC_REGIONS_FILTERING = ( + 'difficult', + 'none', + 'giab_conf', +) + +HOTSPOT_FIELDS_FILTERING = ( + 'SAGE_HOTSPOT', + 'hotspot', + 'PCGR_MUTATION_HOTSPOT', +) + +RETAIN_FIELDS_FILTERING = ( + 'PANEL', + *HOTSPOT_FIELDS_FILTERING, +) + + ################################################## ## VCF FILTER tags and FORMAT, INFO annotations ## ################################################## @@ -61,6 +98,8 @@ class VcfFilter(enum.Enum): ENCODE = 'ENCODE' GNOMAD_COMMON = 'gnomAD_common' + PCGR_COUNT_LIMIT = 'PCGR_count_limit' + @property def namespace(self): return 'FILTER' @@ -121,6 +160,8 @@ class VcfInfo(enum.Enum): RESCUED_FILTERS_EXISTING = 'RESCUED_FILTERS_EXISTING' RESCUED_FILTERS_PENDING = 'RESCUED_FILTERS_PENDING' + PANEL = 'PANEL' + @property def namespace(self): return 'INFO' @@ -187,6 +228,9 @@ def namespace(self): 'Description': f'gnomAD AF >= {MAX_GNOMAD_AF}', }, + VcfFilter.PCGR_COUNT_LIMIT: { + 'Description': 'Manually filtered to meet PCGR 500,000 variant limit', + }, # INFO VcfInfo.TUMOR_AF: { @@ -350,6 +394,12 @@ def namespace(self): 'Description': 'Filters pending prior to variant rescue', }, + VcfInfo.PANEL: { + 'Number': '0', + 'Type': 'Flag', + 'Description': 'UMCCR somatic panel CDS (2,000 bp padding)', + }, + # FORMAT VcfFormat.SAGE_AD: { diff --git a/bolt/common/pcgr.py b/bolt/common/pcgr.py index fce4c56..296fb48 100644 --- a/bolt/common/pcgr.py +++ b/bolt/common/pcgr.py @@ -1,9 +1,12 @@ +import collections import csv +import functools +import itertools import pathlib import re import shutil import tempfile - +import logging import cyvcf2 @@ -11,6 +14,8 @@ from .. import util from ..common import constants +# Use the existing logger configuration +logger = logging.getLogger(__name__) def prepare_vcf_somatic(input_fp, tumor_name, normal_name, output_dir): @@ -110,13 +115,18 @@ def get_minimal_header(input_fh): return '\n'.join([filetype_line, *chrom_lines, *format_lines, column_line]) -def run_somatic(input_fp, pcgr_refdata_dir, output_dir, threads=1, pcgr_conda=None, pcgrr_conda=None, purity=None, ploidy=None, sample_id=None): +def run_somatic(input_fp, pcgr_refdata_dir, pcgr_pcgr_output_dir, chunk_nbr=None, chunk_nbr=None, threads=1, pcgr_conda=None, pcgrr_conda=None, purity=None, ploidy=None, sample_id=None): # NOTE(SW): Nextflow FusionFS v2.2.8 does not support PCGR output to S3; instead write to a # temporary directory outside of the FusionFS mounted directory then manually copy across temp_dir = tempfile.TemporaryDirectory() - pcgr_output_dir = output_dir / 'pcgr/' + temp_dir_path = pathlib.Path(temp_dir.name) + pcgr_output_dir = pcgr_output_dir / f"pcgr_{chunk_nbr}" if chunk_nbr is not None else pcgr_output_dir # Check if the output directory already exists + if pcgr_output_dir.exists(): + logger.warning(f"Warning: Output directory '{pcgr_output_dir}' already exists and will be overwrited") + shutil.rmtree(pcgr_output_dir) + if not sample_id: sample_id = 'nosampleset' @@ -171,7 +181,7 @@ def run_somatic(input_fp, pcgr_refdata_dir, output_dir, threads=1, pcgr_conda=No command = fr''' pcgr \ - {command_args_str} + {command_args_str} ''' if pcgr_conda: @@ -179,11 +189,24 @@ def run_somatic(input_fp, pcgr_refdata_dir, output_dir, threads=1, pcgr_conda=No command_formatting = '\n' + ' ' * 4 command = command_formatting + command_conda + command - util.execute_command(command) + # Log file path + log_file_path = temp_dir_path / "run_somatic.log" + + # Run the command and redirect output to the log file + util.execute_command(command, log_file_path=log_file_path) shutil.copytree(temp_dir.name, pcgr_output_dir) - return pcgr_output_dir + pcgr_tsv_fp = pathlib.Path(pcgr_output_dir) / 'nosampleset.pcgr_acmg.grch38.snvs_indels.tiers.tsv' + pcgr_vcf_fp = pathlib.Path(pcgr_output_dir) / 'nosampleset.pcgr_acmg.grch38.vcf.gz' + + # Check if both files exist + if not pcgr_tsv_fp.exists(): + raise FileNotFoundError(f"Expected file {pcgr_tsv_fp} not found.") + if not pcgr_vcf_fp.exists(): + raise FileNotFoundError(f"Expected file {pcgr_vcf_fp} not found.") + + return pcgr_tsv_fp, pcgr_vcf_fp def run_germline(input_fp, panel_fp, pcgr_refdata_dir, output_dir, threads=1, pcgr_conda=None, pcgrr_conda=None, sample_id=None): @@ -240,7 +263,7 @@ def run_germline(input_fp, panel_fp, pcgr_refdata_dir, output_dir, threads=1, pc return cpsr_output_dir -def transfer_annotations_somatic(input_fp, tumor_name, filter_name, pcgr_dir, output_dir): +def transfer_annotations_somatic(input_fp, tumor_name, pcgr_vcf_fp, pcgr_tsv_fp, output_dir): # Set destination INFO field names and source TSV fields info_field_map = { constants.VcfInfo.PCGR_MUTATION_HOTSPOT: 'MUTATION_HOTSPOT', @@ -249,9 +272,6 @@ def transfer_annotations_somatic(input_fp, tumor_name, filter_name, pcgr_dir, ou constants.VcfInfo.PCGR_CSQ: 'CSQ', } - pcgr_tsv_fp = pathlib.Path(pcgr_dir) / 'nosampleset.pcgr_acmg.grch38.snvs_indels.tiers.tsv' - pcgr_vcf_fp = pathlib.Path(pcgr_dir) / 'nosampleset.pcgr_acmg.grch38.vcf.gz' - # Enforce matching defined and source INFO annotations check_annotation_headers(info_field_map, pcgr_vcf_fp) @@ -277,10 +297,6 @@ def transfer_annotations_somatic(input_fp, tumor_name, filter_name, pcgr_dir, ou # Do not process chrM since *snvs_indels.tiers.tsv does not include these annotations if record.CHROM == 'chrM': continue - # Immediately print out variants that were not annotated - if filter_name in record.FILTERS: - output_fh.write_record(record) - continue # Annotate and write record_ann = annotate_record(record, pcgr_data) output_fh.write_record(record_ann) @@ -481,3 +497,193 @@ def annotate_record(record, annotations, *, allow_missing=False): record.INFO[info_enum.value] = v return record + +def split_vcf(input_vcf, output_dir): + """ + Splits a VCF file into multiple chunks, each containing up to max_variants variants. + Each chunk includes the VCF header. + Ensures no overlapping positions between chunks. + """ + output_dir = pathlib.Path(output_dir / "vcf_chunks") + output_dir.mkdir(parents=True, exist_ok=True) + chunk_files = [] + chunk_number = 1 + variant_count = 0 + base_filename = pathlib.Path(input_vcf).stem + chunk_filename = output_dir / f"{base_filename}_chunk{chunk_number}.vcf" + base_filename = input_vcf.stem + chunk_filename = output_dir / f"{base_filename}_chunk{chunk_number}.vcf" + chunk_files.append(chunk_filename) + # Open the input VCF using cyvcf2 + vcf_in = cyvcf2.VCF(input_vcf) + # Create a new VCF file for the first chunk + vcf_out = cyvcf2.Writer(str(chunk_filename), vcf_in) + last_position = None + for record in vcf_in: + current_position = record.POS + # Check if we need to start a new chunk + if variant_count >= constants.MAX_SOMATIC_VARIANTS and (last_position is None or current_position != last_position): + # Close the current chunk file and start a new one + vcf_out.close() + chunk_number += 1 + chunk_filename = output_dir / f"{base_filename}_chunk{chunk_number}.vcf" + chunk_files.append(chunk_filename) + vcf_out = cyvcf2.Writer(str(chunk_filename), vcf_in) + variant_count = 0 + # Write the record to the current chunk + vcf_out.write_record(record) + variant_count += 1 + last_position = current_position + # Close the last chunk file + vcf_out.close() + vcf_in.close() + logger.info(f"VCF file split into {len(chunk_files)} chunks.") + return chunk_files + +def run_somatic_chunck(vcf_chunks, pcgr_data_dir, output_dir, pcgr_output_dir, max_threads, pcgr_conda, pcgrr_conda): + pcgr_tsv_files = [] + pcgr_vcf_files = [] + num_chunks = len(vcf_chunks) + # Ensure we don't use more workers than available threads, and each worker has at least 2 threads + max_workers = min(num_chunks, max_threads // 2) + threads_quot, threads_rem = divmod(max_threads, num_chunks) + threads_per_chunk = max(2, threads_quot) + # Limit the number of workers to the smaller of num_chunks or max_threads // 2 + with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor: + futures = {} + for chunk_number, vcf_file in enumerate(vcf_chunks, start=1): + # Assign extra thread to the first 'threads_rem' chunks + additional_thread = 1 if chunk_number <= threads_rem else 0 + total_threads = threads_per_chunk + additional_thread + futures[executor.submit(run_somatic, vcf_file, pcgr_data_dir, pcgr_output_dir, chunk_number, total_threads, pcgr_conda, pcgrr_conda)] = chunk_number + for future in concurrent.futures.as_completed(futures): + try: + pcgr_tsv_fp, pcgr_vcf_fp = future.result() + if pcgr_tsv_fp: + pcgr_tsv_files.append(pcgr_tsv_fp) + if pcgr_vcf_fp: + pcgr_vcf_files.append(pcgr_vcf_fp) + except Exception as e: + print(f"Exception occurred: {e}") + merged_vcf_fp, merged_tsv_fp = merging_pcgr_files(output_dir, pcgr_vcf_files, pcgr_tsv_files) + return merged_tsv_fp, merged_vcf_fp + +def merging_pcgr_files(output_dir, pcgr_vcf_files, pcgr_tsv_fp): + pcgr_dir = pathlib.Path(output_dir) / 'pcgr' + pcgr_dir.mkdir(exist_ok=True) + + # Merge all TSV files into a single file in the pcgr directory + merged_tsv_fp = pcgr_dir / "nosampleset.pcgr_acmg.grch38.snvs_indels.tiers.tsv" + util.merge_tsv_files(pcgr_tsv_fp, merged_tsv_fp) + + # Step 5: Merge all VCF files into a single file in the pcgr directory + merged_vcf_path = pcgr_dir / "nosampleset.pcgr_acmg.grch38" + merged_vcf = util.merge_vcf_files(pcgr_vcf_files, merged_vcf_path) + + return merged_vcf, merged_tsv_fp + + +def get_variant_filter_data(variant): + attribute_names = ( + 'tier', + 'difficult', + 'giab_conf', + 'intergenic', + 'intronic', + 'downstream', + 'upstream', + 'impacts_other', + ) + + data = {e: None for e in attribute_names} + + + data['tier'] = variant.INFO['PCGR_TIER'] + + + info_keys = [k for k, v in variant.INFO] + + data['difficult'] = any(e.startswith('DIFFICULT') for e in info_keys) + data['giab_conf'] = 'GIAB_CONF' in info_keys + + + # NOTE(SW): GIAB_CONF always overrides DIFFICULT tags + if data['giab_conf'] and data['difficult']: + data['difficult']= False + + + for impact in get_impacts(variant.INFO['PCGR_CSQ']): + if impact == 'intergenic_variant': + data['intergenic'] = True + elif impact == 'intron_variant': + data['intronic'] = True + elif impact == 'downstream_gene_variant': + data['downstream'] = True + elif impact == 'upstream_gene_variant': + data['upstream'] = True + elif impact: + data['impacts_other'] = True + else: + assert False + + return data + + +def get_impacts(csq_str_full): + impacts = set() + for csq_str in csq_str_full.split(','): + csq_tokens = csq_str.split('|') + impact_str = csq_tokens[1] + impacts.update(impact_str.split('&')) + return impacts + + +def determine_filter(data): + + for impact, region in get_ordering(tiers=False): + + # NOTE(SW): this is less efficient than nested loops since the outer block is reevaluated + # within what would be the inner loop each cycle; taking this route for cleaner code + + impacts_higher = get_impacts_higher(impact) + impact_filter = bool(data[impact]) and not any(bool(data[e]) for e in impacts_higher) + + region_filter = False + if region == 'none': + region_filter = not (data['difficult'] or data['giab_conf']) + else: + region_filter = data[region] + + if impact_filter and region_filter: + return (impact, region) + + return False + + +def get_variant_repr(variant): + return (variant.CHROM, variant.POS, variant.REF, tuple(variant.ALT)) + + +@functools.cache +def get_ordering(tiers=True, impacts=True, regions=True): + categories = [ + constants.PCGR_TIERS_FILTERING if tiers else None, + constants.VEP_IMPACTS_FILTER if impacts else None, + constants.GENOMIC_REGIONS_FILTERING if regions else None, + ] + + # NOTE(SW): I'm not aware of any noncoding impacts for TIER_[1-4] other than TERT but keeping + # in to be overly cautious + ordering_iter = itertools.product(*(c for c in categories if c)) + + return tuple(ordering_iter) + + +@functools.cache +def get_impacts_higher(impact): + impact_index = constants.VEP_IMPACTS_FILTER.index(impact) + if impact_index + 1 < len(constants.VEP_IMPACTS_FILTER): + impacts_higher = constants.VEP_IMPACTS_FILTER[impact_index+1:len(constants.VEP_IMPACTS_FILTER)] + else: + impacts_higher = list() + return impacts_higher diff --git a/bolt/logging_config.py b/bolt/logging_config.py new file mode 100644 index 0000000..0ddda2b --- /dev/null +++ b/bolt/logging_config.py @@ -0,0 +1,34 @@ +import logging +import sys +import pathlib +from datetime import datetime + +class IgnoreTinfoFilter(logging.Filter): + def filter(self, record): + # Exclude messages that contain the unwanted text. + if "no version information available" in record.getMessage(): + return False + return True + +def setup_logging(output_dir, script_name): + # Create a timestamp for the log file + timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') + log_filename = f"{script_name}_{timestamp}.log" + log_file = pathlib.Path(output_dir) / log_filename + + # Create individual handlers. + console_handler = logging.StreamHandler(sys.stdout) + file_handler = logging.FileHandler(log_file) + + # Instantiate and attach the filter to both handlers. + tinfo_filter = IgnoreTinfoFilter() + console_handler.addFilter(tinfo_filter) + file_handler.addFilter(tinfo_filter) + + logging.basicConfig( + level=logging.DEBUG, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + handlers=[file_handler, console_handler] + ) + logger = logging.getLogger(__name__) + logger.info("Logging setup complete") \ No newline at end of file diff --git a/bolt/util.py b/bolt/util.py index b7333d9..b2ce7df 100644 --- a/bolt/util.py +++ b/bolt/util.py @@ -1,11 +1,14 @@ +import os import pathlib import subprocess -import sys import textwrap - +import logging +from types import SimpleNamespace from .common import constants +# Set up logging +logger = logging.getLogger(__name__) # TODO(SW): create note that number this assumes location of `//` def get_project_root(): @@ -15,46 +18,58 @@ def get_project_root(): return project_root -def execute_command(command): - command_prepared = command_prepare(command) +def execute_command(command, log_file_path=None): + logger.info("Executing command: %s", command.strip()) - print(command_prepared) + # Open the log file if provided + log_file = log_file_path.open('a', encoding='utf-8') if log_file_path else None - process = subprocess.run( - command_prepared, + # Launch process with combined stdout and stderr streams, and line buffering enabled. + process = subprocess.Popen( + command, shell=True, executable='/bin/bash', - capture_output=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, encoding='utf-8', + bufsize=1 # line buffered ) - if process.returncode != 0: - print(process) - print(process.stderr) - sys.exit(1) + output_lines = [] + # Iterate over each line as it becomes available + with process.stdout: + for line in iter(process.stdout.readline, ''): + if line: + logger.info(line.strip()) + output_lines.append(line) + if log_file: + log_file.write(line) + log_file.flush() # flush immediately for real-time logging + process.wait() # wait for the process to complete + + if log_file: + log_file.close() - return process + result = SimpleNamespace( + stdout=''.join(output_lines), + returncode=process.returncode, + pid=process.pid, + command=command + ) + return result def command_prepare(command): return f'set -o pipefail; {textwrap.dedent(command)}' - -#def count_vcf_records(fp, exclude_args=None): -# args = list() -# if exclude_args: -# args.append(f'-e \'{exclude_args}\'') -# -# args_str = ' '.join(args) -# command = f'bcftools view -H {args_str} {fp} | wc -l' -# -# result = execute_command(command) -# return int(result.stdout) - - def count_vcf_records(fp): - result = execute_command(f'bcftools view -H {fp} | wc -l') - return int(result.stdout) + result = subprocess.run(f'bcftools view -H {fp} | wc -l', + shell=True, + executable="/bin/bash", + capture_output=True, + text=True ) + return int(result.stdout.strip()) def add_vcf_header_entry(fh, anno_enum): @@ -94,8 +109,185 @@ def get_qualified_vcf_annotation(anno_enum): assert anno_enum in constants.VcfInfo or anno_enum in constants.VcfFormat return f'{anno_enum.namespace}/{anno_enum.value}' +def split_vcf(input_vcf, output_dir): + """ + Splits a VCF file into multiple chunks, each containing up to max_variants variants. + Each chunk includes the VCF header. + Ensures no overlapping positions between chunks. + """ + output_dir = pathlib.Path(output_dir / "vcf_chunks") + output_dir.mkdir(parents=True, exist_ok=True) + + chunk_files = [] + chunk_number = 1 + variant_count = 0 + base_filename = pathlib.Path(input_vcf).stem + chunk_filename = output_dir / f"{base_filename}_chunk{chunk_number}.vcf" + base_filename = input_vcf.stem + chunk_filename = output_dir / f"{base_filename}_chunk{chunk_number}.vcf" + chunk_files.append(chunk_filename) + + # Open the input VCF using cyvcf2 + vcf_in = cyvcf2.VCF(input_vcf) + # Create a new VCF file for the first chunk + vcf_out = cyvcf2.Writer(str(chunk_filename), vcf_in) + + last_position = None + + for record in vcf_in: + current_position = record.POS + # Check if we need to start a new chunk + if variant_count >= constants.MAX_SOMATIC_VARIANTS and (last_position is None or current_position != last_position): + # Close the current chunk file and start a new one + vcf_out.close() + chunk_number += 1 + chunk_filename = output_dir / f"{base_filename}_chunk{chunk_number}.vcf" + chunk_files.append(chunk_filename) + vcf_out = cyvcf2.Writer(str(chunk_filename), vcf_in) + variant_count = 0 + + # Write the record to the current chunk + vcf_out.write_record(record) + variant_count += 1 + last_position = current_position + + # Close the last chunk file + vcf_out.close() + vcf_in.close() + + logger.info(f"VCF file split into {len(chunk_files)} chunks.") + + return chunk_files + +def merge_tsv_files(tsv_files, merged_tsv_fp): + """ + Merges all TSV files into a single TSV. + """ + with open(merged_tsv_fp, 'w') as merged_tsv: + for i, tsv_file in enumerate(tsv_files): + with open(tsv_file, 'r') as infile: + for line_number, line in enumerate(infile): + # Skip header except for the first file + if i > 0 and line_number == 0: + continue + merged_tsv.write(line) + logger.info(f"Merged TSV written to: {merged_tsv_fp}") + + +def merge_vcf_files(vcf_files, merged_vcf_fp): + """ + Merges multiple VCF files into a single sorted VCF file using bcftools. + + Parameters: + - vcf_files: List of paths to VCF files to be merged. + - merged_vcf_fp: Path to the output merged VCF file (without extension). + + Returns: + - Path to the sorted merged VCF file. + """ + merged_vcf_fp = pathlib.Path(merged_vcf_fp) + merged_unsorted_vcf = merged_vcf_fp.with_suffix('.unsorted.vcf.gz') + merged_vcf = merged_vcf_fp.with_suffix('.vcf.gz') + + # Prepare the bcftools merge command arguments + command_args = [ + 'bcftools merge', + '-m all', + '-Oz', + f'-o {merged_unsorted_vcf}', + ] + [str(vcf_file) for vcf_file in vcf_files] + + # Format the command for readability + delimiter_padding = ' ' * 10 + delimiter = f' \\\n{delimiter_padding}' + command_args_str = delimiter.join(command_args) + + command = f''' + {command_args_str} + ''' + + # Run the bcftools merge command + logger.info("Running bcftools merge...") + execute_command(command) + logger.info(f"Merged VCF written to: {merged_unsorted_vcf}") + + # Sort the merged VCF file + sort_command_args = [ + 'bcftools sort', + '-Oz', + f'-o {merged_vcf}', + f'{merged_unsorted_vcf}' + ] + sort_command_args_str = delimiter.join(sort_command_args) + sort_command = f''' + {sort_command_args_str} + ''' + + logger.info("Sorting merged VCF file...") + execute_command(sort_command) + logger.info(f"Sorted merged VCF written to: {merged_vcf}") + + # Index the sorted merged VCF file + index_command_args = [ + 'bcftools index', + '-t', + f'{merged_vcf}' + ] + index_command_args_str = delimiter.join(index_command_args) + index_command = f''' + {index_command_args_str} + ''' + + logger.info("Indexing sorted merged VCF file...") + execute_command(index_command) + logger.info(f"Indexed merged VCF file: {merged_vcf}.tbi") + + # Optionally, remove the unsorted merged VCF file + if merged_unsorted_vcf.exists(): + merged_unsorted_vcf.unlink() + + return merged_vcf + +def merging_pcgr_files(output_dir, pcgr_vcf_files, pcgr_tsv_fp): + # Step 3: Merge all chunk VCF files into a single file + pcgr_dir = output_dir / 'pcgr/' + pcgr_dir.mkdir(exist_ok=True) + # Merge all TSV files into a single file in the pcgr directory merged_tsv_fp = os.path.join(pcgr_dir, "nosampleset.pcgr_acmg.grch38.snvs_indels.tiers.tsv") + merged_tsv_fp = os.path.join(pcgr_dir, "nosampleset.pcgr_acmg.grch38.snvs_indels.tiers.tsv") + merge_tsv_files(pcgr_tsv_fp, merged_tsv_fp) + # Step 5: Merge all VCF files into a single file in the pcgr directory + merged_vcf_path = os.path.join(pcgr_dir, "nosampleset.pcgr_acmg.grch38") + merged_vcf = merge_vcf_files(pcgr_vcf_files, merged_vcf_path) + return merged_vcf, merged_tsv_fp + +def run_somatic_chunck(vcf_chunks, pcgr_data_dir, output_dir, pcgr_output_dir, max_threads, pcgr_conda, pcgrr_conda): + pcgr_tsv_files = [] + pcgr_vcf_files = [] + + num_chunks = len(vcf_chunks) + # Ensure we don't use more workers than available threads, and each worker has at least 2 threads + max_workers = min(num_chunks, max_threads // 2) + threads_quot, threads_rem = divmod(max_threads, num_chunks) + threads_per_chunk = max(2, threads_quot) + + # Limit the number of workers to the smaller of num_chunks or max_threads // 2 + with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor: + futures = {} + for chunk_number, vcf_file in enumerate(vcf_chunks, start=1): + # Assign extra thread to the first 'threads_rem' chunks + additional_thread = 1 if chunk_number <= threads_rem else 0 + total_threads = threads_per_chunk + additional_thread + futures[executor.submit(pcgr.run_somatic, vcf_file, pcgr_data_dir, pcgr_output_dir, chunk_number, total_threads, pcgr_conda, pcgrr_conda)] = chunk_number + + for future in concurrent.futures.as_completed(futures): + try: + pcgr_tsv_fp, pcgr_vcf_fp = future.result() + if pcgr_tsv_fp: + pcgr_tsv_files.append(pcgr_tsv_fp) + if pcgr_vcf_fp: + pcgr_vcf_files.append(pcgr_vcf_fp) + except Exception as e: + print(f"Exception occurred: {e}") -#def add_vcf_filter(record, filter_enum): -# existing_filters = [e for e in record.FILTERS if e != 'PASS'] -# assert filter_enum.value not in existing_filters -# return ';'.join([*existing_filters, filter_enum.value]) + merged_vcf_fp, merged_tsv_fp = merging_pcgr_files(output_dir, pcgr_vcf_files, pcgr_tsv_files) + return merged_tsv_fp, merged_vcf_fp \ No newline at end of file diff --git a/bolt/workflows/other/cancer_report.py b/bolt/workflows/other/cancer_report.py index 60b4da3..1902079 100644 --- a/bolt/workflows/other/cancer_report.py +++ b/bolt/workflows/other/cancer_report.py @@ -5,6 +5,9 @@ from ... import util +from ...logging_config import setup_logging + + @click.command(name='cancer_report') @@ -48,6 +51,9 @@ def entry(ctx, **kwargs): output_dir = pathlib.Path(kwargs['output_dir']) output_dir.mkdir(mode=0o755, parents=True, exist_ok=True) + script_name = pathlib.Path(__file__).stem + setup_logging(output_dir, script_name) + # Normalise SAGE variants and remove duplicates that arise for MutationalPattern compatibility decomposed_snv_vcf = normalise_and_dedup_sage_variants( kwargs['smlv_somatic_vcf_fp'], diff --git a/bolt/workflows/smlv_germline/report.py b/bolt/workflows/smlv_germline/report.py index 0742c19..0a10711 100644 --- a/bolt/workflows/smlv_germline/report.py +++ b/bolt/workflows/smlv_germline/report.py @@ -1,5 +1,6 @@ import pathlib import yaml +from ...logging_config import setup_logging import click @@ -35,6 +36,9 @@ def entry(ctx, **kwargs): output_dir = pathlib.Path(kwargs['output_dir']) output_dir.mkdir(mode=0o755, parents=True, exist_ok=True) + # Set up logging + script_name = pathlib.Path(__file__).stem + setup_logging(output_dir, script_name) # BCFtools stats run_bcftool_stats(kwargs['vcf_unfiltered_fp'], kwargs['normal_name'], output_dir) diff --git a/bolt/workflows/smlv_somatic/annotate.py b/bolt/workflows/smlv_somatic/annotate.py index f163d5f..fe0007a 100644 --- a/bolt/workflows/smlv_somatic/annotate.py +++ b/bolt/workflows/smlv_somatic/annotate.py @@ -1,14 +1,13 @@ +import logging import pathlib - - import click import cyvcf2 - from ... import util from ...common import constants from ...common import pcgr - +from ...logging_config import setup_logging +logger = logging.getLogger(__name__) @click.command(name='annotate') @click.pass_context @@ -44,6 +43,9 @@ def entry(ctx, **kwargs): output_dir = pathlib.Path(kwargs['output_dir']) output_dir.mkdir(mode=0o755, parents=True, exist_ok=True) + script_name = pathlib.Path(__file__).stem + setup_logging(output_dir, script_name) + # Set all FILTER="." to FILTER="PASS" as required by PURPLE filter_pass_fp = set_filter_pass(kwargs['vcf_fp'], kwargs['tumor_name'], output_dir) @@ -73,7 +75,6 @@ def entry(ctx, **kwargs): ) # Annotate with cancer-related and functional information from a range of sources using PCGR - # - Select variants to process - there is an upper limit for PCGR of around 500k # - Set tumor and normal AF and DP in INFO for PCGR and remove all other annotations # - Run PCGR on minimal VCF (pcgr_prep_fp) # - Transfer selected PCGR annotations to unfiltered VCF (selected_fp) @@ -84,30 +85,40 @@ def entry(ctx, **kwargs): # - Hits in COSMIC [INFO/PCGR_COSMIC_COUNT] # - Hits in TCGA [INFO/PCGR_TCGA_PANCANCER_COUNT] # - Hits in PCAWG [INFO/PCGR_ICGC_PCAWG_COUNT] - # Set selected data or full input - selection_data = select_variants( - pon_fp, - kwargs['tumor_name'], - kwargs['cancer_genes_fp'], - output_dir, - ) - - if not (pcgr_prep_input_fp := selection_data.get('filtered')): - pcgr_prep_input_fp = selection_data['selected'] # Prepare VCF for PCGR annotation pcgr_prep_fp = pcgr.prepare_vcf_somatic( - pcgr_prep_input_fp, + pon_fp, kwargs['tumor_name'], kwargs['normal_name'], output_dir, ) - # Run PCGR - pcgr_dir = pcgr.run_somatic( - pcgr_prep_fp, + pcgr_output_dir = output_dir / 'pcgr' + total_variants = util.count_vcf_records(pcgr_prep_fp) + print(f"Total number of variants in the input VCF: {total_variants}") + + # Run PCGR in chunks if the total number of variants exceeds the maximum allowed for somatic variants + if total_variants > constants.MAX_SOMATIC_VARIANTS: + vcf_chunks = util.split_vcf( + pcgr_prep_fp, + output_dir + ) + pcgr_tsv_fp, pcgr_vcf_fp = util.run_somatic_chunck( + vcf_chunks, kwargs['pcgr_data_dir'], output_dir, + pcgr_output_dir, + kwargs['threads'], + kwargs['pcgr_conda'], + kwargs['pcgrr_conda'] + ) + else: + pcgr_tsv_fp, pcgr_vcf_fp = pcgr.run_somatic( + pcgr_prep_fp, + kwargs['pcgr_data_dir'], + pcgr_output_dir, + chunk_nbr=None, threads=kwargs['threads'], pcgr_conda=kwargs['pcgr_conda'], pcgrr_conda=kwargs['pcgrr_conda'], @@ -115,13 +126,13 @@ def entry(ctx, **kwargs): # Transfer PCGR annotations to full set of variants pcgr.transfer_annotations_somatic( - selection_data['selected'], + pon_fp, kwargs['tumor_name'], - selection_data.get('filter_name'), - pcgr_dir, + pcgr_vcf_fp, + pcgr_tsv_fp, output_dir, ) - + logger.info("Annotation process completed") def set_filter_pass(input_fp, tumor_name, output_dir): output_fp = output_dir / f'{tumor_name}.set_filter_pass.vcf.gz' @@ -136,7 +147,6 @@ def set_filter_pass(input_fp, tumor_name, output_dir): return output_fp - def general_annotations(input_fp, tumor_name, threads, annotations_dir, output_dir): toml_fp = pathlib.Path(annotations_dir) / 'vcfanno_annotations.toml' @@ -169,7 +179,6 @@ def panel_of_normal_annotations(input_fp, tumor_name, threads, pon_dir, output_d util.execute_command(command) return output_fp - def select_variants(input_fp, tumor_name, cancer_genes_fp, output_dir): # Exclude variants until we hopefully move the needle below the threshold diff --git a/bolt/workflows/smlv_somatic/filter.py b/bolt/workflows/smlv_somatic/filter.py index e46f285..104296d 100644 --- a/bolt/workflows/smlv_somatic/filter.py +++ b/bolt/workflows/smlv_somatic/filter.py @@ -26,6 +26,9 @@ def entry(ctx, **kwargs): output_dir = pathlib.Path(kwargs['output_dir']) output_dir.mkdir(mode=0o755, parents=True, exist_ok=True) + script_name = pathlib.Path(__file__).stem + setup_logging(output_dir, script_name) + # Open input VCF and set required header entries for output in_fh = cyvcf2.VCF(kwargs['vcf_fp']) header_filters = ( diff --git a/bolt/workflows/smlv_somatic/report.py b/bolt/workflows/smlv_somatic/report.py index bdf908f..b2bead1 100644 --- a/bolt/workflows/smlv_somatic/report.py +++ b/bolt/workflows/smlv_somatic/report.py @@ -1,3 +1,4 @@ +import collections import csv import json import pathlib @@ -45,6 +46,9 @@ def entry(ctx, **kwargs): output_dir = pathlib.Path(kwargs['output_dir']) output_dir.mkdir(mode=0o755, parents=True, exist_ok=True) + script_name = pathlib.Path(__file__).stem + setup_logging(output_dir, script_name) + # BCFtools stats bcftools_vcf_fp = bcftools_stats_prepare(kwargs['vcf_fp'], kwargs['tumor_name'], output_dir) run_bcftools_stats(bcftools_vcf_fp, kwargs['tumor_name'], output_dir) @@ -105,10 +109,20 @@ def entry(ctx, **kwargs): fh.write('\n') # PCGR report + if variant_counts_process['filter_pass'] <= constants.MAX_SOMATIC_VARIANTS: + pcgr_input_vcf_fp = kwargs['vcf_fp'] + else: + pcgr_input_vcf_fp = select_pcgr_variants( + kwargs['vcf_fp'], + kwargs['cancer_genes_fp'], + kwargs['tumor_name'], + output_dir, + ) + purple_data = parse_purple_purity_file(kwargs['purple_purity_fp']) pcgr_prep_fp = pcgr.prepare_vcf_somatic( - kwargs['vcf_fp'], + pcgr_input_vcf_fp, kwargs['tumor_name'], kwargs['normal_name'], output_dir, @@ -284,6 +298,75 @@ def count_variant_process(vcf_fp): return counts +def select_pcgr_variants(vcf_fp, cancer_genes_fp, tumor_name, output_dir): + # Annotate variants in UMCCR somatic gene panel + fp_annotated_out = output_dir / f'{tumor_name}.umccr_panel_variants_annotated.vcf.gz' + util.execute_command(fr''' + bcftools annotate \ + --annotations <(awk 'BEGIN {{ OFS="\t" }} {{ print $1, $2-2000, $3+2000, "1" }}' {cancer_genes_fp}) \ + --header-line '{util.get_vcf_header_line(constants.VcfInfo.PANEL)}' \ + --columns CHROM,FROM,TO,{constants.VcfInfo.PANEL.value} \ + --output {fp_annotated_out} \ + {vcf_fp} + ''') + + # Set filter category for each variant + variants_sorted = collections.defaultdict(list) + for variant_count, variant in enumerate(cyvcf2.VCF(fp_annotated_out), 1): + if any(variant.INFO.get(e) for e in constants.RETAIN_FIELDS_FILTERING): + continue + + data = pcgr.get_variant_filter_data(variant) + variant_filter = pcgr.determine_filter(data) + assert variant_filter + + filter_category = (data['tier'], *variant_filter) + variant_repr = pcgr.get_variant_repr(variant) + variants_sorted[filter_category].append(variant_repr) + + + # Determine the set of filter categories to come under the PCGR 500,000 variant threshold + filter_sum = 0 + filter_categories = list() + for key in pcgr.get_ordering(): + + if (variant_count - filter_sum) <= constants.MAX_SOMATIC_VARIANTS: + break + + filter_sum += len(variants_sorted.get(key, [])) + filter_categories.append(key) + + # Set FILTERS and write out records + filter_variants = set() + for key in filter_categories: + filter_variants.update(variants_sorted[key]) + + fh_in = cyvcf2.VCF(vcf_fp) + util.add_vcf_header_entry(fh_in, constants.VcfFilter.PCGR_COUNT_LIMIT) + + # NOTE(SW): creating an additional VCF with all records for traceability + fp_out = output_dir / f'{tumor_name}.pcgr_hypermutated.pass.vcf.gz' + fp_set_out = output_dir / f'{tumor_name}.pcgr_hypermutated.filters_set.vcf.gz' + + fh_out = cyvcf2.Writer(fp_out, fh_in) + fh_set_out = cyvcf2.Writer(fp_set_out, fh_in) + + for variant in fh_in: + variant_repr = pcgr.get_variant_repr(variant) + if variant_repr not in filter_variants: + # Write only passing + fh_out.write_record(variant) + else: + variant.FILTER = constants.VcfFilter.PCGR_COUNT_LIMIT.value + # Write all variants including those with FILTER set + fh_set_out.write_record(variant) + + fh_out.close() + fh_set_out.close() + + return fp_out + + def parse_purple_purity_file(fp): with open(fp, 'r') as fh: entries = list(csv.DictReader(fh, delimiter='\t')) diff --git a/bolt/workflows/smlv_somatic/rescue.py b/bolt/workflows/smlv_somatic/rescue.py index c92df94..bfd89b9 100644 --- a/bolt/workflows/smlv_somatic/rescue.py +++ b/bolt/workflows/smlv_somatic/rescue.py @@ -7,10 +7,14 @@ import click import cyvcf2 +import logging from ... import util from ...common import constants +from ...logging_config import setup_logging + +logger = logging.getLogger(__name__) @click.command(name='rescue') @@ -37,6 +41,9 @@ def entry(ctx, **kwargs): output_dir = pathlib.Path(kwargs['output_dir']) output_dir.mkdir(mode=0o755, parents=True, exist_ok=True) + script_name = pathlib.Path(__file__).stem + setup_logging(output_dir, script_name) + # Select PASS SAGE variants in hotspots and then split into existing and novel calls sage_pass_vcf_fp = select_sage_pass_hotspot( kwargs['sage_vcf_fp'], diff --git a/bolt/workflows/sv_somatic/annotate.py b/bolt/workflows/sv_somatic/annotate.py index 2988e53..8577922 100644 --- a/bolt/workflows/sv_somatic/annotate.py +++ b/bolt/workflows/sv_somatic/annotate.py @@ -5,10 +5,13 @@ import click import cyvcf2 import pysam +import logging from ... import util +logger = logging.getLogger(__name__) + @click.command(name='annotate') @click.pass_context diff --git a/conda/env/bolt_env.yml b/conda/env/bolt_env.yml index e27bd24..71e90c9 100644 --- a/conda/env/bolt_env.yml +++ b/conda/env/bolt_env.yml @@ -16,3 +16,4 @@ dependencies: - python >=3.10 - pyyaml - vcfanno ==0.3.5 + - ncurses>=6.3 diff --git a/pyproject.toml b/pyproject.toml index e7325a3..e98f470 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ dependencies = [ "cyvcf2", "pysam", "pyyaml", + "future" ] [project.scripts] From cb5ee4d510acd85db1061a65a0c42a0806ae430f Mon Sep 17 00:00:00 2001 From: Quentin Clayssen <37511834+qclayssen@users.noreply.github.com> Date: Tue, 2 Sep 2025 15:31:24 +1000 Subject: [PATCH 03/89] Feature/pcgr v2.2.1 (#10) --- bolt/common/constants.py | 78 +++++------- bolt/common/pcgr.py | 152 +++++++++++++----------- bolt/util.py | 1 + bolt/workflows/smlv_germline/report.py | 2 + bolt/workflows/smlv_somatic/annotate.py | 16 ++- bolt/workflows/smlv_somatic/filter.py | 16 +-- bolt/workflows/smlv_somatic/report.py | 4 +- docker/Dockerfile.pcgr | 4 +- tests/test_smlv_somatic_filter.py | 12 +- 9 files changed, 145 insertions(+), 140 deletions(-) diff --git a/bolt/common/constants.py b/bolt/common/constants.py index 29216bc..7cebe78 100644 --- a/bolt/common/constants.py +++ b/bolt/common/constants.py @@ -35,9 +35,9 @@ 'pathogenic', 'uncertain_significance', } -PCGR_TIERS_RESCUE = { - 'TIER_1', - 'TIER_2', +PCGR_ACTIONABILITY_TIER_RESCUE = { + '1', + '2', } @@ -116,16 +116,14 @@ class VcfInfo(enum.Enum): SAGE_NOVEL = 'SAGE_NOVEL' SAGE_RESCUE = 'SAGE_RESCUE' - PCGR_TIER = 'PCGR_TIER' + PCGR_ACTIONABILITY_TIER = 'PCGR_ACTIONABILITY_TIER' PCGR_CSQ = 'PCGR_CSQ' PCGR_MUTATION_HOTSPOT = 'PCGR_MUTATION_HOTSPOT' - PCGR_CLINVAR_CLNSIG = 'PCGR_CLINVAR_CLNSIG' + PCGR_CLINVAR_CLASSIFICATION = 'PCGR_CLINVAR_CLASSIFICATION' PCGR_COSMIC_COUNT = 'PCGR_COSMIC_COUNT' PCGR_TCGA_PANCANCER_COUNT = 'PCGR_TCGA_PANCANCER_COUNT' PCGR_ICGC_PCAWG_COUNT = 'PCGR_ICGC_PCAWG_COUNT' - CPSR_FINAL_CLASSIFICATION = 'CPSR_FINAL_CLASSIFICATION' - CPSR_PATHOGENICITY_SCORE = 'CPSR_PATHOGENICITY_SCORE' CPSR_CLINVAR_CLASSIFICATION = 'CPSR_CLINVAR_CLASSIFICATION' CPSR_CSQ = 'CPSR_CSQ' @@ -151,7 +149,7 @@ class VcfInfo(enum.Enum): GNOMAD_AF = 'gnomAD_AF' - PCGR_TIER_RESCUE = 'PCGR_TIER_RESCUE' + PCGR_ACTIONABILITY_TIER_RESCUE = 'PCGR_ACTIONABILITY_TIER_RESCUE' SAGE_HOTSPOT_RESCUE = 'SAGE_HOTSPOT_RESCUE' CLINICAL_POTENTIAL_RESCUE = 'CLINICAL_POTENTIAL_RESCUE' @@ -270,7 +268,7 @@ def namespace(self): 'Description': 'Variant rescued by a matching SAGE call', }, - VcfInfo.PCGR_TIER: { + VcfInfo.PCGR_ACTIONABILITY_TIER: { 'Number': '1', 'Type': 'String', 'Description': ( @@ -281,28 +279,29 @@ def namespace(self): }, VcfInfo.PCGR_CSQ: { 'Number': '.', - 'Type': 'String', - 'Description': ( - 'Consequence annotations from Ensembl VEP. Format: Allele|Consequence|IMPACT|SYMBOL|' - 'Gene|Feature_type|Feature|BIOTYPE|EXON|INTRON|HGVSc|HGVSp|cDNA_position|' - 'CDS_position|Protein_position|Amino_acids|Codons|Existing_variation|ALLELE_NUM|' - 'DISTANCE|STRAND|FLAGS|PICK|VARIANT_CLASS|SYMBOL_SOURCE|HGNC_ID|CANONICAL|' - 'MANE_SELECT|MANE_PLUS_CLINICAL|TSL|APPRIS|CCDS|ENSP|SWISSPROT|TREMBL|UNIPARC|' - 'UNIPROT_ISOFORM|RefSeq|DOMAINS|HGVS_OFFSET|AF|AFR_AF|AMR_AF|EAS_AF|EUR_AF|SAS_AF|' - 'gnomAD_AF|gnomAD_AFR_AF|gnomAD_AMR_AF|gnomAD_ASJ_AF|gnomAD_EAS_AF|gnomAD_FIN_AF|' - 'gnomAD_NFE_AF|gnomAD_OTH_AF|gnomAD_SAS_AF|CLIN_SIG|SOMATIC|PHENO|CHECK_REF|' - 'NearestExonJB' - ), + 'Type': 'String', + 'Description': ( + 'Consequence annotations from Ensembl VEP. Format: ' + 'Allele|Consequence|IMPACT|SYMBOL|Gene|Feature_type|Feature|BIOTYPE|EXON|INTRON|HGVSc|' + 'HGVSp|cDNA_position|CDS_position|Protein_position|Amino_acids|Codons|Existing_variation|' + 'ALLELE_NUM|DISTANCE|STRAND|FLAGS|PICK|VARIANT_CLASS|SYMBOL_SOURCE|HGNC_ID|CANONICAL|' + 'MANE|MANE_SELECT|MANE_PLUS_CLINICAL|TSL|APPRIS|CCDS|ENSP|SWISSPROT|TREMBL|UNIPARC|' + 'UNIPROT_ISOFORM|RefSeq|DOMAINS|HGVS_OFFSET|gnomADe_AF|gnomADe_AFR_AF|gnomADe_AMR_AF|' + 'gnomADe_ASJ_AF|gnomADe_EAS_AF|gnomADe_FIN_AF|gnomADe_MID_AF|gnomADe_NFE_AF|' + 'gnomADe_REMAINING_AF|gnomADe_SAS_AF|CLIN_SIG|SOMATIC|PHENO|CHECK_REF|MOTIF_NAME|' + 'MOTIF_POS|HIGH_INF_POS|MOTIF_SCORE_CHANGE|TRANSCRIPTION_FACTORS|NearestExonJB|' + 'MaxEntScan_alt|MaxEntScan_diff|MaxEntScan_ref' + ), }, VcfInfo.PCGR_MUTATION_HOTSPOT: { 'Number': '.', 'Type': 'String', - 'Description': 'Known cancer mutation hotspot, as found in cancerhotspots.org_v2, Gene|Codon|Q-value', + 'Description': 'Known cancer mutation hotspot, as found in cancerhotspots.org. Format: GeneSymbol|Entrez_ID|CodonRefAA|Alt_AA|Q-value', }, - VcfInfo.PCGR_CLINVAR_CLNSIG: { + VcfInfo.PCGR_CLINVAR_CLASSIFICATION: { 'Number': '.', 'Type': 'String', - 'Description': 'ClinVar clinical significance', + 'Description': 'ClinVar - clinical significance - per phenotype submission', }, VcfInfo.PCGR_COSMIC_COUNT: { 'Number': '1', @@ -320,23 +319,10 @@ def namespace(self): 'Description': 'Count of ICGC PCAWG hits', }, - VcfInfo.CPSR_FINAL_CLASSIFICATION: { - 'Number': '1', - 'Type': 'String', - 'Description': ( - 'Final variant classification based on the combination of CLINVAR_CLASSIFICTION (for ' - 'ClinVar-classified variants), and CPSR_CLASSIFICATION (for novel variants)' - ), - }, - VcfInfo.CPSR_PATHOGENICITY_SCORE: { - 'Number': '1', - 'Type': 'Float', - 'Description': 'Aggregated CPSR pathogenicity score', - }, VcfInfo.CPSR_CLINVAR_CLASSIFICATION: { 'Number': '1', 'Type': 'String', - 'Description': 'Clinical significance of variant on a five-tiered scale', + 'Description': 'ClinVar - Overall clinical significance of variant on a five-tiered scale', }, VcfInfo.CPSR_CSQ: { 'Number': '.', @@ -345,13 +331,13 @@ def namespace(self): 'Consequence annotations from Ensembl VEP. Format: Allele|Consequence|IMPACT|SYMBOL|' 'Gene|Feature_type|Feature|BIOTYPE|EXON|INTRON|HGVSc|HGVSp|cDNA_position|CDS_position|' 'Protein_position|Amino_acids|Codons|Existing_variation|ALLELE_NUM|DISTANCE|STRAND|' - 'FLAGS|PICK|VARIANT_CLASS|SYMBOL_SOURCE|HGNC_ID|CANONICAL|MANE_SELECT|' - 'MANE_PLUS_CLINICAL|APPRIS|CCDS|ENSP|SWISSPROT|TREMBL|UNIPARC|UNIPROT_ISOFORM|RefSeq|' - 'DOMAINS|HGVS_OFFSET|AF|AFR_AF|AMR_AF|EAS_AF|EUR_AF|SAS_AF|gnomAD_AF|gnomAD_AFR_AF|' - 'gnomAD_AMR_AF|gnomAD_ASJ_AF|gnomAD_EAS_AF|gnomAD_FIN_AF|gnomAD_NFE_AF|gnomAD_OTH_AF|' - 'gnomAD_SAS_AF|CLIN_SIG|SOMATIC|PHENO|CHECK_REF|MOTIF_NAME|MOTIF_POS|HIGH_INF_POS|' - 'MOTIF_SCORE_CHANGE|TRANSCRIPTION_FACTORS|NearestExonJB|LoF|LoF_filter|LoF_flags|' - 'LoF_info' + 'FLAGS|PICK|VARIANT_CLASS|SYMBOL_SOURCE|HGNC_ID|CANONICAL|MANE|MANE_SELECT|' + 'MANE_PLUS_CLINICAL|TSL|APPRIS|CCDS|ENSP|SWISSPROT|TREMBL|UNIPARC|UNIPROT_ISOFORM|RefSeq|' + 'DOMAINS|HGVS_OFFSET|gnomADe_AF|gnomADe_AFR_AF|gnomADe_AMR_AF|gnomADe_ASJ_AF|' + 'gnomADe_EAS_AF|gnomADe_FIN_AF|gnomADe_MID_AF|gnomADe_NFE_AF|gnomADe_REMAINING_AF|' + 'gnomADe_SAS_AF|CLIN_SIG|SOMATIC|PHENO|CHECK_REF|MOTIF_NAME|MOTIF_POS|HIGH_INF_POS|' + 'MOTIF_SCORE_CHANGE|TRANSCRIPTION_FACTORS|NearestExonJB|MaxEntScan_alt|MaxEntScan_diff|' + 'MaxEntScan_ref' ), }, @@ -360,7 +346,7 @@ def namespace(self): 'Type': 'Flag', 'Description': '', }, - VcfInfo.PCGR_TIER_RESCUE: { + VcfInfo.PCGR_ACTIONABILITY_TIER_RESCUE: { 'Number': '0', 'Type': 'Flag', 'Description': '', diff --git a/bolt/common/pcgr.py b/bolt/common/pcgr.py index 296fb48..8a17865 100644 --- a/bolt/common/pcgr.py +++ b/bolt/common/pcgr.py @@ -1,6 +1,7 @@ import collections import csv import functools +import gzip import itertools import pathlib import re @@ -115,18 +116,17 @@ def get_minimal_header(input_fh): return '\n'.join([filetype_line, *chrom_lines, *format_lines, column_line]) -def run_somatic(input_fp, pcgr_refdata_dir, pcgr_pcgr_output_dir, chunk_nbr=None, chunk_nbr=None, threads=1, pcgr_conda=None, pcgrr_conda=None, purity=None, ploidy=None, sample_id=None): +def run_somatic(input_fp, pcgr_refdata_dir, vep_dir, output_dir, chunk_nbr=None, threads=1, pcgr_conda=None, pcgrr_conda=None, purity=None, ploidy=None, sample_id=None): - # NOTE(SW): Nextflow FusionFS v2.2.8 does not support PCGR output to S3; instead write to a - # temporary directory outside of the FusionFS mounted directory then manually copy across - temp_dir = tempfile.TemporaryDirectory() - temp_dir_path = pathlib.Path(temp_dir.name) - pcgr_output_dir = pcgr_output_dir / f"pcgr_{chunk_nbr}" if chunk_nbr is not None else pcgr_output_dir # Check if the output directory already exists - if pcgr_output_dir.exists(): - logger.warning(f"Warning: Output directory '{pcgr_output_dir}' already exists and will be overwrited") - shutil.rmtree(pcgr_output_dir) + output_dir = output_dir / f"pcgr_{chunk_nbr}" if chunk_nbr is not None else output_dir + if output_dir.exists(): + logger.warning(f"Output directory '{output_dir}' already exists and will be overwritten") + shutil.rmtree(output_dir) + + # Create output directory + output_dir.mkdir(parents=True, exist_ok=True) if not sample_id: sample_id = 'nosampleset' @@ -134,19 +134,20 @@ def run_somatic(input_fp, pcgr_refdata_dir, pcgr_pcgr_output_dir, chunk_nbr=None command_args = [ f'--sample_id {sample_id}', f'--input_vcf {input_fp}', + f'--vep_dir {vep_dir}', + f'--refdata_dir {pcgr_refdata_dir}', f'--tumor_dp_tag TUMOR_DP', f'--tumor_af_tag TUMOR_AF', f'--control_dp_tag NORMAL_DP', f'--control_af_tag NORMAL_AF', - f'--pcgr_dir {pcgr_refdata_dir}', f'--genome_assembly grch38', f'--assay WGS', f'--estimate_signatures', - f'--estimate_msi_status', + f'--estimate_msi', f'--estimate_tmb', - f'--show_noncoding', f'--vcfanno_n_proc {threads}', - f'--vep_pick_order biotype,rank,appris,tsl,ccds,canonical,length,mane', + f'--vep_n_forks 4', + f'--vep_pick_order biotype,rank,appris,tsl,ccds,canonical,length,mane_plus_clinical,mane_select', ] # NOTE(SW): VEP pick order is applied as a successive filter: @@ -172,7 +173,7 @@ def run_somatic(input_fp, pcgr_refdata_dir, pcgr_pcgr_output_dir, chunk_nbr=None command_args.append(f'--tumor_ploidy {ploidy}') # NOTE(SW): placed here to always have output directory last - command_args.append(f'--output_dir {temp_dir.name}') + command_args.append(f'--output_dir {output_dir}') delimiter_padding = ' ' * 10 delimiter = f' \\\n{delimiter_padding}' @@ -190,18 +191,16 @@ def run_somatic(input_fp, pcgr_refdata_dir, pcgr_pcgr_output_dir, chunk_nbr=None command = command_formatting + command_conda + command # Log file path - log_file_path = temp_dir_path / "run_somatic.log" + log_file_path = output_dir / "run_somatic.log" # Run the command and redirect output to the log file util.execute_command(command, log_file_path=log_file_path) - shutil.copytree(temp_dir.name, pcgr_output_dir) - - pcgr_tsv_fp = pathlib.Path(pcgr_output_dir) / 'nosampleset.pcgr_acmg.grch38.snvs_indels.tiers.tsv' - pcgr_vcf_fp = pathlib.Path(pcgr_output_dir) / 'nosampleset.pcgr_acmg.grch38.vcf.gz' + pcgr_tsv_fp = pathlib.Path(output_dir) / f'{sample_id}.pcgr.grch38.snv_indel_ann.tsv.gz' + pcgr_vcf_fp = pathlib.Path(output_dir) / f'{sample_id}.pcgr.grch38.pass.vcf.gz' # Check if both files exist - if not pcgr_tsv_fp.exists(): + if not pcgr_tsv_fp.exists(): raise FileNotFoundError(f"Expected file {pcgr_tsv_fp} not found.") if not pcgr_vcf_fp.exists(): raise FileNotFoundError(f"Expected file {pcgr_vcf_fp} not found.") @@ -209,37 +208,42 @@ def run_somatic(input_fp, pcgr_refdata_dir, pcgr_pcgr_output_dir, chunk_nbr=None return pcgr_tsv_fp, pcgr_vcf_fp -def run_germline(input_fp, panel_fp, pcgr_refdata_dir, output_dir, threads=1, pcgr_conda=None, pcgrr_conda=None, sample_id=None): +def run_germline(input_fp, panel_fp, pcgr_refdata_dir, vep_dir, output_dir, threads=1, pcgr_conda=None, pcgrr_conda=None, sample_id=None): if not sample_id: sample_id = 'nosampleset' - # NOTE(SW): Nextflow FusionFS v2.2.8 does not support PCGR output to S3; instead write to a - # temporary directory outside of the FusionFS mounted directory then manually copy across - temp_dir = tempfile.TemporaryDirectory() cpsr_output_dir = output_dir / 'cpsr/' + if cpsr_output_dir.exists(): + logger.warning(f"Output directory '{cpsr_output_dir}' already exists and will be overwritten") + shutil.rmtree(cpsr_output_dir) + + # Create output directory + cpsr_output_dir.mkdir(parents=True, exist_ok=True) + command_args = [ f'--sample_id {sample_id}', f'--input_vcf {input_fp}', f'--genome_assembly grch38', f'--custom_list {panel_fp}', + f'--vep_dir {pcgr_refdata_dir}', + f'--refdata_dir {pcgr_refdata_dir}', # NOTE(SW): probably useful to add versioning information here; weigh against maintainence # burden f'--custom_list_name umccr_germline_panel', f'--pop_gnomad global', f'--classify_all', - f'--pcgr_dir {pcgr_refdata_dir}', f'--vcfanno_n_proc {threads}', - f'--vep_pick_order biotype,rank,appris,tsl,ccds,canonical,length,mane', + f'--vep_pick_order biotype,rank,appris,tsl,ccds,canonical,length,mane_plus_clinical,mane_select', ] if pcgrr_conda: command_args.append(f'--pcgrr_conda {pcgrr_conda}') # NOTE(SW): placed here to always have output directory last - command_args.append(f'--output_dir {temp_dir.name}') + command_args.append(f'--output_dir {cpsr_output_dir}') delimiter_padding = ' ' * 10 delimiter = f' \\\n{delimiter_padding}' @@ -258,8 +262,6 @@ def run_germline(input_fp, panel_fp, pcgr_refdata_dir, output_dir, threads=1, pc util.execute_command(command) - shutil.copytree(temp_dir.name, cpsr_output_dir) - return cpsr_output_dir @@ -267,11 +269,14 @@ def transfer_annotations_somatic(input_fp, tumor_name, pcgr_vcf_fp, pcgr_tsv_fp, # Set destination INFO field names and source TSV fields info_field_map = { constants.VcfInfo.PCGR_MUTATION_HOTSPOT: 'MUTATION_HOTSPOT', - constants.VcfInfo.PCGR_CLINVAR_CLNSIG: 'CLINVAR_CLNSIG', + constants.VcfInfo.PCGR_CLINVAR_CLASSIFICATION: 'CLINVAR_CLASSIFICATION', constants.VcfInfo.PCGR_TCGA_PANCANCER_COUNT: 'TCGA_PANCANCER_COUNT', constants.VcfInfo.PCGR_CSQ: 'CSQ', } + pcgr_tsv_fp = pathlib.Path(output_dir) / 'nosampleset.pcgr_acmg.grch38.snvs_indels.tiers.tsv' + pcgr_vcf_fp = pathlib.Path(output_dir) / 'nosampleset.pcgr_acmg.grch38.vcf.gz' + # Enforce matching defined and source INFO annotations check_annotation_headers(info_field_map, pcgr_vcf_fp) @@ -281,13 +286,11 @@ def transfer_annotations_somatic(input_fp, tumor_name, pcgr_vcf_fp, pcgr_tsv_fp, # Open filehandles, set required header entries input_fh = cyvcf2.VCF(input_fp) - util.add_vcf_header_entry(input_fh, constants.VcfInfo.PCGR_TIER) + util.add_vcf_header_entry(input_fh, constants.VcfInfo.PCGR_ACTIONABILITY_TIER) util.add_vcf_header_entry(input_fh, constants.VcfInfo.PCGR_CSQ) util.add_vcf_header_entry(input_fh, constants.VcfInfo.PCGR_MUTATION_HOTSPOT) - util.add_vcf_header_entry(input_fh, constants.VcfInfo.PCGR_CLINVAR_CLNSIG) - util.add_vcf_header_entry(input_fh, constants.VcfInfo.PCGR_COSMIC_COUNT) + util.add_vcf_header_entry(input_fh, constants.VcfInfo.PCGR_CLINVAR_CLASSIFICATION) util.add_vcf_header_entry(input_fh, constants.VcfInfo.PCGR_TCGA_PANCANCER_COUNT) - util.add_vcf_header_entry(input_fh, constants.VcfInfo.PCGR_ICGC_PCAWG_COUNT) output_fp = output_dir / f'{tumor_name}.annotations.vcf.gz' output_fh = cyvcf2.Writer(output_fp, input_fh, 'wz') @@ -298,20 +301,19 @@ def transfer_annotations_somatic(input_fp, tumor_name, pcgr_vcf_fp, pcgr_tsv_fp, if record.CHROM == 'chrM': continue # Annotate and write - record_ann = annotate_record(record, pcgr_data) + record_ann = annotate_record(record, pcgr_data, allow_missing=True) output_fh.write_record(record_ann) def transfer_annotations_germline(input_fp, normal_name, cpsr_dir, output_dir): # Set destination INFO field names and source TSV fields + # Note: Only include fields that exist in CPSR v2.2.1 output info_field_map = { - constants.VcfInfo.CPSR_FINAL_CLASSIFICATION: 'FINAL_CLASSIFICATION', - constants.VcfInfo.CPSR_PATHOGENICITY_SCORE: 'CPSR_PATHOGENICITY_SCORE', constants.VcfInfo.CPSR_CLINVAR_CLASSIFICATION: 'CLINVAR_CLASSIFICATION', constants.VcfInfo.CPSR_CSQ: 'CSQ', } - cpsr_tsv_fp = pathlib.Path(cpsr_dir) / f'{normal_name}.cpsr.grch38.snvs_indels.tiers.tsv' + cpsr_tsv_fp = pathlib.Path(cpsr_dir) / f'{normal_name}.cpsr.grch38.classification.tsv.gz' cpsr_vcf_fp = pathlib.Path(cpsr_dir) / f'{normal_name}.cpsr.grch38.vcf.gz' # Enforce matching defined and source INFO annotations @@ -323,8 +325,6 @@ def transfer_annotations_germline(input_fp, normal_name, cpsr_dir, output_dir): # Open filehandles, set required header entries input_fh = cyvcf2.VCF(input_fp) - util.add_vcf_header_entry(input_fh, constants.VcfInfo.CPSR_FINAL_CLASSIFICATION) - util.add_vcf_header_entry(input_fh, constants.VcfInfo.CPSR_PATHOGENICITY_SCORE) util.add_vcf_header_entry(input_fh, constants.VcfInfo.CPSR_CLINVAR_CLASSIFICATION) util.add_vcf_header_entry(input_fh, constants.VcfInfo.CPSR_CSQ) @@ -352,13 +352,14 @@ def check_annotation_headers(info_field_map, vcf_fp): try: header_src_entry = vcf_fh.get_header_type(header_src) except KeyError: + print(f"Missing in VCF: {header_src}") continue header_dst_entry = util.get_vcf_header_entry(header_dst) # Remove leading and trailing quotes from source header_src_description_unquoted = header_src_entry['Description'].strip('"') - assert header_src_description_unquoted == header_dst_entry['Description'] - + if header_src_description_unquoted != header_dst_entry['Description']: + raise AssertionError(f"Mismatch for {header_src}:\nVCF: {header_src_description_unquoted}\nExpected: {header_dst_entry['Description']}") def collect_pcgr_annotation_data(tsv_fp, vcf_fp, info_field_map): # Gather all annotations from TSV @@ -368,27 +369,13 @@ def collect_pcgr_annotation_data(tsv_fp, vcf_fp, info_field_map): key, record_ann = get_annotation_entry_tsv(record, info_field_map) assert key not in data_tsv - # Process PCGR_TIER - # TIER_1, TIER_2, TIER_3, TIER_4, NONCODING - record_ann[constants.VcfInfo.PCGR_TIER] = record['TIER'].replace(' ', '_') - - # Count COSMIC hits - if record['COSMIC_MUTATION_ID'] == 'NA': - cosmic_count = 0 - else: - cosmic_count = len(record['COSMIC_MUTATION_ID'].split('&')) - record_ann[constants.VcfInfo.PCGR_COSMIC_COUNT] = cosmic_count - - # Count ICGC-PCAWG hits by taking sum of affected donors where the annotation value has - # the following format: project_code|tumor_type|affected_donors|tested_donors|frequency - icgc_pcawg_count = 0 - if record['ICGC_PCAWG_OCCURRENCE'] != 'NA': - for pcawg_hit_data in record['ICGC_PCAWG_OCCURRENCE'].split(','): - pcawrg_hit_data_fields = pcawg_hit_data.split('|') - affected_donors = int(pcawrg_hit_data_fields[2]) - icgc_pcawg_count += affected_donors - assert icgc_pcawg_count > 0 - record_ann[constants.VcfInfo.PCGR_ICGC_PCAWG_COUNT] = icgc_pcawg_count + # Process PCGR_ACTIONABILITY_TIER + # TIER 1, TIER 2, TIER 3, TIER 4, NONCODING + record_ann[constants.VcfInfo.PCGR_ACTIONABILITY_TIER] = record['ACTIONABILITY_TIER'].replace(' ', '_') + + # Process PCGR_ACTIONABILITY_TIER + # TIER 1, TIER 2, TIER 3, TIER 4, NONCODING + record_ann[constants.VcfInfo.PCGR_ACTIONABILITY_TIER] = record['ACTIONABILITY_TIER'] # Store annotation data data_tsv[key] = record_ann @@ -404,7 +391,7 @@ def collect_cpsr_annotation_data(tsv_fp, vcf_fp, info_field_map): # Gather annotations from TSV data_tsv = dict() gdot_re = re.compile('^(?P[\dXYM]+):g\.(?P\d+)(?P[A-Z]+)>(?P[A-Z]+)$') - with open(tsv_fp, 'r') as tsv_fh: + with gzip.open(tsv_fp, 'rt') as tsv_fh: for record in csv.DictReader(tsv_fh, delimiter='\t'): # Decompose CPSR 'GENOMIC_CHANGE' field into CHROM, POS, REF, and ALT re_result = gdot_re.match(record['GENOMIC_CHANGE']) @@ -426,6 +413,24 @@ def collect_cpsr_annotation_data(tsv_fp, vcf_fp, info_field_map): # Compile annotations, prefering TSV source return compile_annotation_data(data_tsv, data_vcf) +def parse_genomic_change(genomic_change): + """ + Parse a genomic change string, e.g., "3:g.41224645T>C" + Returns a tuple: (chrom, pos, ref, alt) + """ + # Regular expression for the format "chrom:g.posRef>Alt" + pattern = r'^(?P\w+):g\.(?P\d+)(?P\w+)>(?P\w+)$' + match = re.match(pattern, genomic_change) + if not match: + raise ValueError(f"Format not recognized: {genomic_change}") + + # Get values and format as needed + chrom = f"chr{match.group('chrom')}" + pos = int(match.group('pos')) + ref = match.group('ref') + alt = match.group('alt') + return chrom, pos, ref, alt + def get_annotations_vcf(vcf_fp, info_field_map): data_vcf = dict() @@ -445,10 +450,14 @@ def get_annotations_vcf(vcf_fp, info_field_map): def get_annotation_entry_tsv(record, info_field_map): - # Set lookup key; PCGR/CPSR strips leading 'chr' from contig names - chrom = f'chr{record["CHROM"]}' - pos = int(record['POS']) - key = (chrom, pos, record['REF'], record['ALT']) + # If GENOMIC_CHANGE is present, parse it for coordinates; otherwise, use separate fields. + if "GENOMIC_CHANGE" in record and record["GENOMIC_CHANGE"]: + chrom, pos, ref, alt = parse_genomic_change(record["GENOMIC_CHANGE"]) + + if not chrom.startswith('chr'): + chrom = f'chr{chrom}' + + key = (chrom, pos, ref, alt) record_ann = dict() for info_dst, info_src in info_field_map.items(): @@ -510,7 +519,6 @@ def split_vcf(input_vcf, output_dir): chunk_number = 1 variant_count = 0 base_filename = pathlib.Path(input_vcf).stem - chunk_filename = output_dir / f"{base_filename}_chunk{chunk_number}.vcf" base_filename = input_vcf.stem chunk_filename = output_dir / f"{base_filename}_chunk{chunk_number}.vcf" chunk_files.append(chunk_filename) @@ -540,7 +548,7 @@ def split_vcf(input_vcf, output_dir): logger.info(f"VCF file split into {len(chunk_files)} chunks.") return chunk_files -def run_somatic_chunck(vcf_chunks, pcgr_data_dir, output_dir, pcgr_output_dir, max_threads, pcgr_conda, pcgrr_conda): +def run_somatic_chunck(vcf_chunks, pcgr_data_dir, vep_dir, output_dir, pcgr_output_dir, max_threads, pcgr_conda, pcgrr_conda): pcgr_tsv_files = [] pcgr_vcf_files = [] num_chunks = len(vcf_chunks) @@ -555,7 +563,7 @@ def run_somatic_chunck(vcf_chunks, pcgr_data_dir, output_dir, pcgr_output_dir, m # Assign extra thread to the first 'threads_rem' chunks additional_thread = 1 if chunk_number <= threads_rem else 0 total_threads = threads_per_chunk + additional_thread - futures[executor.submit(run_somatic, vcf_file, pcgr_data_dir, pcgr_output_dir, chunk_number, total_threads, pcgr_conda, pcgrr_conda)] = chunk_number + futures[executor.submit(run_somatic, vcf_file, pcgr_data_dir, vep_dir, pcgr_output_dir, chunk_number, total_threads, pcgr_conda, pcgrr_conda)] = chunk_number for future in concurrent.futures.as_completed(futures): try: pcgr_tsv_fp, pcgr_vcf_fp = future.result() @@ -577,7 +585,7 @@ def merging_pcgr_files(output_dir, pcgr_vcf_files, pcgr_tsv_fp): util.merge_tsv_files(pcgr_tsv_fp, merged_tsv_fp) # Step 5: Merge all VCF files into a single file in the pcgr directory - merged_vcf_path = pcgr_dir / "nosampleset.pcgr_acmg.grch38" + merged_vcf_path = pcgr_dir / "nosampleset.pcgr.grch38.pass.vcf.gz" merged_vcf = util.merge_vcf_files(pcgr_vcf_files, merged_vcf_path) return merged_vcf, merged_tsv_fp diff --git a/bolt/util.py b/bolt/util.py index b2ce7df..34df80f 100644 --- a/bolt/util.py +++ b/bolt/util.py @@ -1,3 +1,4 @@ +import gzip import os import pathlib import subprocess diff --git a/bolt/workflows/smlv_germline/report.py b/bolt/workflows/smlv_germline/report.py index 0a10711..395268f 100644 --- a/bolt/workflows/smlv_germline/report.py +++ b/bolt/workflows/smlv_germline/report.py @@ -23,6 +23,7 @@ @click.option('--germline_panel_list_fp', required=True, type=click.Path(exists=True)) @click.option('--pcgr_data_dir', required=True, type=click.Path(exists=True)) +@click.option('--vep_dir', required=True, type=click.Path(exists=True)) @click.option('--threads', required=True, type=int, default=1) @@ -73,6 +74,7 @@ def entry(ctx, **kwargs): cpsr_prep_fp, kwargs['germline_panel_list_fp'], kwargs['pcgr_data_dir'], + kwargs['vep_dir'], output_dir, threads=kwargs['threads'], pcgr_conda=kwargs['pcgr_conda'], diff --git a/bolt/workflows/smlv_somatic/annotate.py b/bolt/workflows/smlv_somatic/annotate.py index fe0007a..471b816 100644 --- a/bolt/workflows/smlv_somatic/annotate.py +++ b/bolt/workflows/smlv_somatic/annotate.py @@ -22,6 +22,7 @@ @click.option('--pon_dir', required=True, type=click.Path(exists=True)) @click.option('--pcgr_data_dir', required=True, type=click.Path(exists=True)) +@click.option('--vep_dir', required=True, type=click.Path(exists=True)) @click.option('--pcgr_conda', required=False, type=str) @click.option('--pcgrr_conda', required=False, type=str) @@ -81,8 +82,7 @@ def entry(ctx, **kwargs): # - PCGR ACMG TIER [INFO/PCGR_TIER] # - VEP consequence [INFO/PCR_CSQ] # - Known mutation hotspot [INFO/PCGR_MUTATION_HOTSPOT] - # - ClinVar clinical significant [INFO/PCGR_CLINVAR_CLNSIG] - # - Hits in COSMIC [INFO/PCGR_COSMIC_COUNT] + # - ClinVar clinical significant [INFO/PCGR_CLNSIG] # - Hits in TCGA [INFO/PCGR_TCGA_PANCANCER_COUNT] # - Hits in PCAWG [INFO/PCGR_ICGC_PCAWG_COUNT] @@ -107,6 +107,7 @@ def entry(ctx, **kwargs): pcgr_tsv_fp, pcgr_vcf_fp = util.run_somatic_chunck( vcf_chunks, kwargs['pcgr_data_dir'], + kwargs['vep_dir'], output_dir, pcgr_output_dir, kwargs['threads'], @@ -117,6 +118,17 @@ def entry(ctx, **kwargs): pcgr_tsv_fp, pcgr_vcf_fp = pcgr.run_somatic( pcgr_prep_fp, kwargs['pcgr_data_dir'], + output_dir, + pcgr_output_dir, + kwargs['threads'], + kwargs['pcgr_conda'], + kwargs['pcgrr_conda'] + ) + else: + pcgr_tsv_fp, pcgr_vcf_fp = pcgr.run_somatic( + pcgr_prep_fp, + kwargs['pcgr_data_dir'], + kwargs['vep_dir'], pcgr_output_dir, chunk_nbr=None, threads=kwargs['threads'], diff --git a/bolt/workflows/smlv_somatic/filter.py b/bolt/workflows/smlv_somatic/filter.py index 104296d..814863c 100644 --- a/bolt/workflows/smlv_somatic/filter.py +++ b/bolt/workflows/smlv_somatic/filter.py @@ -40,7 +40,7 @@ def entry(ctx, **kwargs): constants.VcfFilter.ENCODE, constants.VcfFilter.GNOMAD_COMMON, constants.VcfInfo.SAGE_HOTSPOT_RESCUE, - constants.VcfInfo.PCGR_TIER_RESCUE, + constants.VcfInfo.PCGR_ACTIONABILITY_TIER_RESCUE, constants.VcfInfo.CLINICAL_POTENTIAL_RESCUE, constants.VcfInfo.RESCUED_FILTERS_EXISTING, constants.VcfInfo.RESCUED_FILTERS_PENDING, @@ -165,9 +165,9 @@ def set_filter_data(record, tumor_index): ## # PCGR tier rescue ## - pcgr_tier = record.INFO.get(constants.VcfInfo.PCGR_TIER.value) - if pcgr_tier in constants.PCGR_TIERS_RESCUE: - info_rescue.append(constants.VcfInfo.PCGR_TIER_RESCUE) + pcgr_tier = record.INFO.get(constants.VcfInfo.PCGR_ACTIONABILITY_TIER.value) + if pcgr_tier in constants.PCGR_ACTIONABILITY_TIER_RESCUE: + info_rescue.append(constants.VcfInfo.PCGR_ACTIONABILITY_TIER_RESCUE) ## # SAGE hotspot rescue @@ -184,19 +184,15 @@ def set_filter_data(record, tumor_index): # single CLINICAL_POTENTIAL_RESCUE flag # Get ClinVar clinical significance entries - clinvar_clinsig = record.INFO.get(constants.VcfInfo.PCGR_CLINVAR_CLNSIG.value, '') + clinvar_clinsig = record.INFO.get(constants.VcfInfo.PCGR_CLINVAR_CLASSIFICATION.value, '') clinvar_clinsigs = clinvar_clinsig.split(',') # Hit counts in relevant reference somatic mutation databases - cosmic_count = record.INFO.get(constants.VcfInfo.PCGR_COSMIC_COUNT.value, 0) tcga_pancancer_count = record.INFO.get(constants.VcfInfo.PCGR_TCGA_PANCANCER_COUNT.value, 0) - icgc_pcawg_count = record.INFO.get(constants.VcfInfo.PCGR_ICGC_PCAWG_COUNT.value, 0) if ( record.INFO.get(constants.VcfInfo.HMF_HOTSPOT.value) is not None or record.INFO.get(constants.VcfInfo.PCGR_MUTATION_HOTSPOT.value) is not None or any(e in clinvar_clinsigs for e in constants.CLINVAR_CLINSIGS_RESCUE) or - cosmic_count >= constants.MIN_COSMIC_COUNT_RESCUE or - tcga_pancancer_count >= constants.MIN_TCGA_PANCANCER_COUNT_RESCUE or - icgc_pcawg_count >= constants.MIN_ICGC_PCAWG_COUNT_RESCUE + tcga_pancancer_count >= constants.MIN_TCGA_PANCANCER_COUNT_RESCUE ): info_rescue.append(constants.VcfInfo.CLINICAL_POTENTIAL_RESCUE) diff --git a/bolt/workflows/smlv_somatic/report.py b/bolt/workflows/smlv_somatic/report.py index b2bead1..ede90ca 100644 --- a/bolt/workflows/smlv_somatic/report.py +++ b/bolt/workflows/smlv_somatic/report.py @@ -28,6 +28,7 @@ @click.option('--pcgrr_conda', required=False, type=str) @click.option('--pcgr_data_dir', required=False, type=str) +@click.option('--vep_dir', required=True, type=click.Path(exists=True)) @click.option('--purple_purity_fp', required=True, type=click.Path(exists=True)) @click.option('--cancer_genes_fp', required=True, type=click.Path(exists=True)) @@ -131,7 +132,8 @@ def entry(ctx, **kwargs): pcgr.run_somatic( pcgr_prep_fp, kwargs['pcgr_data_dir'], - output_dir, + kwargs['vep_dir'], + pcgr_output_dir, threads=kwargs['threads'], pcgr_conda=kwargs['pcgr_conda'], pcgrr_conda=kwargs['pcgrr_conda'], diff --git a/docker/Dockerfile.pcgr b/docker/Dockerfile.pcgr index 4ea4932..ca984e8 100644 --- a/docker/Dockerfile.pcgr +++ b/docker/Dockerfile.pcgr @@ -15,13 +15,13 @@ RUN \ conda create \ --solver libmamba \ --name pcgr \ - --file https://raw.githubusercontent.com/sigven/pcgr/v1.4.1/conda/env/lock/pcgr-linux-64.lock + --file https://raw.githubusercontent.com/sigven/pcgr/refs/tags/v2.2.1/conda/env/lock/pcgr-linux-64.lock RUN \ conda create \ --solver libmamba \ --name pcgrr \ - --file https://raw.githubusercontent.com/sigven/pcgr/v1.4.1/conda/env/lock/pcgrr-linux-64.lock + --file https://raw.githubusercontent.com/sigven/pcgr/refs/tags/v2.2.1/conda/env/lock/pcgrr-linux-64.lock COPY ./conda/env/bolt_env.yml /tmp/ RUN \ diff --git a/tests/test_smlv_somatic_filter.py b/tests/test_smlv_somatic_filter.py index 41a0e96..6455162 100644 --- a/tests/test_smlv_somatic_filter.py +++ b/tests/test_smlv_somatic_filter.py @@ -160,15 +160,15 @@ def test_common_population_filter(self): def test_pcgr_tier_rescue(self): pcgr_tiers = [ - 'TIER_1', - 'TIER_2', + '1', + '2', ] - rescue_tag_str = bolt_constants.VcfInfo.PCGR_TIER_RESCUE.value + rescue_tag_str = bolt_constants.VcfInfo.PCGR_ACTIONABILITY_TIER_RESCUE.value for pcgr_tier in pcgr_tiers: record = get_record( **self.records['filter_min_af9.9'], - info_data={'PCGR_TIER': pcgr_tier}, + info_data={'PCGR_ACTIONABILITY_TIER': pcgr_tier}, ) smlv_somatic_filter.set_filter_data(record, 0) assert not record.FILTER @@ -189,9 +189,7 @@ def test_clinical_potential_rescue_general(self): info_data_sets = [ {'HMF_HOTSPOT': ''}, {'PCGR_MUTATION_HOTSPOT': ''}, - {'PCGR_COSMIC_COUNT': 11}, {'PCGR_TCGA_PANCANCER_COUNT': 6}, - {'PCGR_ICGC_PCAWG_COUNT': 6}, ] rescue_tag_str = bolt_constants.VcfInfo.CLINICAL_POTENTIAL_RESCUE.value @@ -217,7 +215,7 @@ def test_clinical_potential_rescue_clinvar_clinsig(self): for clinsig in clinsigs: record = get_record( **self.records['filter_min_af9.9'], - info_data={'PCGR_CLINVAR_CLNSIG': clinsig}, + info_data={'PCGR_CLINVAR_CLASSIFICATION': clinsig}, ) smlv_somatic_filter.set_filter_data(record, 0) assert not record.FILTER From 9d8e4d11292d5875dda0dbc901eda0940903bdb4 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Tue, 16 Sep 2025 10:34:44 +1000 Subject: [PATCH 04/89] fix merge duplication and imports --- bolt/common/pcgr.py | 55 ++++++++++++++++--------- bolt/util.py | 51 ++--------------------- bolt/workflows/smlv_somatic/annotate.py | 53 +++++++++--------------- pyproject.toml | 1 - 4 files changed, 59 insertions(+), 101 deletions(-) diff --git a/bolt/common/pcgr.py b/bolt/common/pcgr.py index 8a17865..72376fc 100644 --- a/bolt/common/pcgr.py +++ b/bolt/common/pcgr.py @@ -8,6 +8,7 @@ import shutil import tempfile import logging +import concurrent.futures import cyvcf2 @@ -228,7 +229,7 @@ def run_germline(input_fp, panel_fp, pcgr_refdata_dir, vep_dir, output_dir, thre f'--input_vcf {input_fp}', f'--genome_assembly grch38', f'--custom_list {panel_fp}', - f'--vep_dir {pcgr_refdata_dir}', + f'--vep_dir {vep_dir}', f'--refdata_dir {pcgr_refdata_dir}', # NOTE(SW): probably useful to add versioning information here; weigh against maintainence # burden @@ -274,8 +275,9 @@ def transfer_annotations_somatic(input_fp, tumor_name, pcgr_vcf_fp, pcgr_tsv_fp, constants.VcfInfo.PCGR_CSQ: 'CSQ', } - pcgr_tsv_fp = pathlib.Path(output_dir) / 'nosampleset.pcgr_acmg.grch38.snvs_indels.tiers.tsv' - pcgr_vcf_fp = pathlib.Path(output_dir) / 'nosampleset.pcgr_acmg.grch38.vcf.gz' + # Respect paths provided; do not override + pcgr_tsv_fp = pathlib.Path(pcgr_tsv_fp) + pcgr_vcf_fp = pathlib.Path(pcgr_vcf_fp) # Enforce matching defined and source INFO annotations check_annotation_headers(info_field_map, pcgr_vcf_fp) @@ -364,18 +366,27 @@ def check_annotation_headers(info_field_map, vcf_fp): def collect_pcgr_annotation_data(tsv_fp, vcf_fp, info_field_map): # Gather all annotations from TSV data_tsv = dict() - with open(tsv_fp, 'r') as tsv_fh: + # Read gz or plain text based on extension + open_fn = gzip.open if str(tsv_fp).endswith('.gz') else open + with open_fn(tsv_fp, 'rt') as tsv_fh: for record in csv.DictReader(tsv_fh, delimiter='\t'): key, record_ann = get_annotation_entry_tsv(record, info_field_map) assert key not in data_tsv - # Process PCGR_ACTIONABILITY_TIER - # TIER 1, TIER 2, TIER 3, TIER 4, NONCODING - record_ann[constants.VcfInfo.PCGR_ACTIONABILITY_TIER] = record['ACTIONABILITY_TIER'].replace(' ', '_') - - # Process PCGR_ACTIONABILITY_TIER - # TIER 1, TIER 2, TIER 3, TIER 4, NONCODING - record_ann[constants.VcfInfo.PCGR_ACTIONABILITY_TIER] = record['ACTIONABILITY_TIER'] + # Normalize PCGR actionability tier to simple values: '1','2','3','4','N' + raw_tier = (record.get('ACTIONABILITY_TIER') or '').strip() + tier_norm = raw_tier.replace('_', ' ').upper() + if tier_norm in ('TIER 1','TIER1','1'): + tier_val = '1' + elif tier_norm in ('TIER 2','TIER2','2'): + tier_val = '2' + elif tier_norm in ('TIER 3','TIER3','3'): + tier_val = '3' + elif tier_norm in ('TIER 4','TIER4','4'): + tier_val = '4' + else: + tier_val = 'N' + record_ann[constants.VcfInfo.PCGR_ACTIONABILITY_TIER] = tier_val # Store annotation data data_tsv[key] = record_ann @@ -450,12 +461,18 @@ def get_annotations_vcf(vcf_fp, info_field_map): def get_annotation_entry_tsv(record, info_field_map): - # If GENOMIC_CHANGE is present, parse it for coordinates; otherwise, use separate fields. - if "GENOMIC_CHANGE" in record and record["GENOMIC_CHANGE"]: - chrom, pos, ref, alt = parse_genomic_change(record["GENOMIC_CHANGE"]) - - if not chrom.startswith('chr'): - chrom = f'chr{chrom}' + # If GENOMIC_CHANGE is present, parse it; otherwise, fallback to CHROM/POS/REF/ALT fields + chrom = pos = ref = alt = None + if 'GENOMIC_CHANGE' in record and record['GENOMIC_CHANGE']: + chrom, pos, ref, alt = parse_genomic_change(record['GENOMIC_CHANGE']) + else: + chrom = record.get('CHROM') or record.get('Chromosome') + pos = int(record.get('POS') or record.get('Start_position')) + ref = record.get('REF') + alt = record.get('ALT') + # Ensure chrom has 'chr' prefix + if chrom and not str(chrom).startswith('chr'): + chrom = f'chr{chrom}' key = (chrom, pos, ref, alt) @@ -518,7 +535,7 @@ def split_vcf(input_vcf, output_dir): chunk_files = [] chunk_number = 1 variant_count = 0 - base_filename = pathlib.Path(input_vcf).stem + input_vcf = pathlib.Path(input_vcf) base_filename = input_vcf.stem chunk_filename = output_dir / f"{base_filename}_chunk{chunk_number}.vcf" chunk_files.append(chunk_filename) @@ -606,7 +623,7 @@ def get_variant_filter_data(variant): data = {e: None for e in attribute_names} - data['tier'] = variant.INFO['PCGR_TIER'] + data['tier'] = variant.INFO.get('PCGR_ACTIONABILITY_TIER') info_keys = [k for k, v in variant.INFO] diff --git a/bolt/util.py b/bolt/util.py index 34df80f..7310b89 100644 --- a/bolt/util.py +++ b/bolt/util.py @@ -1,11 +1,11 @@ -import gzip -import os import pathlib import subprocess import textwrap import logging from types import SimpleNamespace +import cyvcf2 + from .common import constants # Set up logging @@ -122,8 +122,7 @@ def split_vcf(input_vcf, output_dir): chunk_files = [] chunk_number = 1 variant_count = 0 - base_filename = pathlib.Path(input_vcf).stem - chunk_filename = output_dir / f"{base_filename}_chunk{chunk_number}.vcf" + input_vcf = pathlib.Path(input_vcf) base_filename = input_vcf.stem chunk_filename = output_dir / f"{base_filename}_chunk{chunk_number}.vcf" chunk_files.append(chunk_filename) @@ -248,47 +247,3 @@ def merge_vcf_files(vcf_files, merged_vcf_fp): merged_unsorted_vcf.unlink() return merged_vcf - -def merging_pcgr_files(output_dir, pcgr_vcf_files, pcgr_tsv_fp): - # Step 3: Merge all chunk VCF files into a single file - pcgr_dir = output_dir / 'pcgr/' - pcgr_dir.mkdir(exist_ok=True) - # Merge all TSV files into a single file in the pcgr directory merged_tsv_fp = os.path.join(pcgr_dir, "nosampleset.pcgr_acmg.grch38.snvs_indels.tiers.tsv") - merged_tsv_fp = os.path.join(pcgr_dir, "nosampleset.pcgr_acmg.grch38.snvs_indels.tiers.tsv") - merge_tsv_files(pcgr_tsv_fp, merged_tsv_fp) - # Step 5: Merge all VCF files into a single file in the pcgr directory - merged_vcf_path = os.path.join(pcgr_dir, "nosampleset.pcgr_acmg.grch38") - merged_vcf = merge_vcf_files(pcgr_vcf_files, merged_vcf_path) - return merged_vcf, merged_tsv_fp - -def run_somatic_chunck(vcf_chunks, pcgr_data_dir, output_dir, pcgr_output_dir, max_threads, pcgr_conda, pcgrr_conda): - pcgr_tsv_files = [] - pcgr_vcf_files = [] - - num_chunks = len(vcf_chunks) - # Ensure we don't use more workers than available threads, and each worker has at least 2 threads - max_workers = min(num_chunks, max_threads // 2) - threads_quot, threads_rem = divmod(max_threads, num_chunks) - threads_per_chunk = max(2, threads_quot) - - # Limit the number of workers to the smaller of num_chunks or max_threads // 2 - with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor: - futures = {} - for chunk_number, vcf_file in enumerate(vcf_chunks, start=1): - # Assign extra thread to the first 'threads_rem' chunks - additional_thread = 1 if chunk_number <= threads_rem else 0 - total_threads = threads_per_chunk + additional_thread - futures[executor.submit(pcgr.run_somatic, vcf_file, pcgr_data_dir, pcgr_output_dir, chunk_number, total_threads, pcgr_conda, pcgrr_conda)] = chunk_number - - for future in concurrent.futures.as_completed(futures): - try: - pcgr_tsv_fp, pcgr_vcf_fp = future.result() - if pcgr_tsv_fp: - pcgr_tsv_files.append(pcgr_tsv_fp) - if pcgr_vcf_fp: - pcgr_vcf_files.append(pcgr_vcf_fp) - except Exception as e: - print(f"Exception occurred: {e}") - - merged_vcf_fp, merged_tsv_fp = merging_pcgr_files(output_dir, pcgr_vcf_files, pcgr_tsv_files) - return merged_tsv_fp, merged_vcf_fp \ No newline at end of file diff --git a/bolt/workflows/smlv_somatic/annotate.py b/bolt/workflows/smlv_somatic/annotate.py index 471b816..d0efa93 100644 --- a/bolt/workflows/smlv_somatic/annotate.py +++ b/bolt/workflows/smlv_somatic/annotate.py @@ -98,43 +98,30 @@ def entry(ctx, **kwargs): total_variants = util.count_vcf_records(pcgr_prep_fp) print(f"Total number of variants in the input VCF: {total_variants}") - # Run PCGR in chunks if the total number of variants exceeds the maximum allowed for somatic variants + # Run PCGR in chunks if exceeding the maximum allowed for somatic variants if total_variants > constants.MAX_SOMATIC_VARIANTS: - vcf_chunks = util.split_vcf( - pcgr_prep_fp, - output_dir + vcf_chunks = pcgr.split_vcf(pcgr_prep_fp, output_dir) + pcgr_tsv_fp, pcgr_vcf_fp = pcgr.run_somatic_chunck( + vcf_chunks, + kwargs['pcgr_data_dir'], + kwargs['vep_dir'], + output_dir, + pcgr_output_dir, + kwargs['threads'], + kwargs['pcgr_conda'], + kwargs['pcgrr_conda'], ) - pcgr_tsv_fp, pcgr_vcf_fp = util.run_somatic_chunck( - vcf_chunks, - kwargs['pcgr_data_dir'], - kwargs['vep_dir'], - output_dir, - pcgr_output_dir, - kwargs['threads'], - kwargs['pcgr_conda'], - kwargs['pcgrr_conda'] - ) - else: - pcgr_tsv_fp, pcgr_vcf_fp = pcgr.run_somatic( - pcgr_prep_fp, - kwargs['pcgr_data_dir'], - output_dir, - pcgr_output_dir, - kwargs['threads'], - kwargs['pcgr_conda'], - kwargs['pcgrr_conda'] - ) else: pcgr_tsv_fp, pcgr_vcf_fp = pcgr.run_somatic( - pcgr_prep_fp, - kwargs['pcgr_data_dir'], - kwargs['vep_dir'], - pcgr_output_dir, - chunk_nbr=None, - threads=kwargs['threads'], - pcgr_conda=kwargs['pcgr_conda'], - pcgrr_conda=kwargs['pcgrr_conda'], - ) + pcgr_prep_fp, + kwargs['pcgr_data_dir'], + kwargs['vep_dir'], + pcgr_output_dir, + chunk_nbr=None, + threads=kwargs['threads'], + pcgr_conda=kwargs['pcgr_conda'], + pcgrr_conda=kwargs['pcgrr_conda'], + ) # Transfer PCGR annotations to full set of variants pcgr.transfer_annotations_somatic( diff --git a/pyproject.toml b/pyproject.toml index e98f470..e7325a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,6 @@ dependencies = [ "cyvcf2", "pysam", "pyyaml", - "future" ] [project.scripts] From 8db35b30bd3c6134314905d68fab6f66250c3457 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Thu, 2 Oct 2025 14:15:26 +1000 Subject: [PATCH 05/89] bump pcgr version --- docker/Dockerfile.pcgr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.pcgr b/docker/Dockerfile.pcgr index ca984e8..8615e50 100644 --- a/docker/Dockerfile.pcgr +++ b/docker/Dockerfile.pcgr @@ -39,7 +39,7 @@ RUN \ RUN \ conda clean -afy -FROM quay.io/bioconda/base-glibc-busybox-bash:2.1.0 +FROM quay.io/bioconda/base-glibc-busybox-bash:2.2.5 # Copy Conda install and all environments COPY --from=build /opt/conda/ /opt/conda/ From c67b30ad4d5ecfa9e28817a61bc341d8f0c71847 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Thu, 2 Oct 2025 14:19:01 +1000 Subject: [PATCH 06/89] bump gpgr version --- docker/Dockerfile.gpgr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.gpgr b/docker/Dockerfile.gpgr index 8b3a157..6b76ecb 100644 --- a/docker/Dockerfile.gpgr +++ b/docker/Dockerfile.gpgr @@ -18,7 +18,7 @@ RUN \ RUN \ conda install --prefix /env/ \ - 'r-gpgr ==2.2.1' \ + 'r-gpgr ==2.2.11' \ 'r-sigrap ==0.1.1' \ 'bioconductor-bsgenome.hsapiens.ucsc.hg38 ==1.4.5' \ 'bioconductor-txdb.hsapiens.ucsc.hg38.knowngene ==3.16.0' \ From f9a7d3b9ce7912b41f6ef97fe42302201bf91c3f Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 17 Oct 2025 09:26:19 +1100 Subject: [PATCH 07/89] change print to logging --- bolt/common/pcgr.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bolt/common/pcgr.py b/bolt/common/pcgr.py index 72376fc..423a8b5 100644 --- a/bolt/common/pcgr.py +++ b/bolt/common/pcgr.py @@ -588,8 +588,9 @@ def run_somatic_chunck(vcf_chunks, pcgr_data_dir, vep_dir, output_dir, pcgr_outp pcgr_tsv_files.append(pcgr_tsv_fp) if pcgr_vcf_fp: pcgr_vcf_files.append(pcgr_vcf_fp) - except Exception as e: - print(f"Exception occurred: {e}") + except Exception: + chunk_number = futures[future] + logger.exception(f"Exception occurred while processing PCGR chunk {chunk_number}.") merged_vcf_fp, merged_tsv_fp = merging_pcgr_files(output_dir, pcgr_vcf_files, pcgr_tsv_files) return merged_tsv_fp, merged_vcf_fp From b8ee4a0b1e2b54aaf6b99d2f3e7571448db8c3c9 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 17 Oct 2025 10:02:11 +1100 Subject: [PATCH 08/89] add argument chunck size --- bolt/common/pcgr.py | 9 +++++++-- bolt/workflows/smlv_somatic/annotate.py | 10 ++++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/bolt/common/pcgr.py b/bolt/common/pcgr.py index 423a8b5..54c87aa 100644 --- a/bolt/common/pcgr.py +++ b/bolt/common/pcgr.py @@ -524,12 +524,17 @@ def annotate_record(record, annotations, *, allow_missing=False): return record -def split_vcf(input_vcf, output_dir): +def split_vcf(input_vcf, output_dir, *, max_variants=None): """ Splits a VCF file into multiple chunks, each containing up to max_variants variants. Each chunk includes the VCF header. Ensures no overlapping positions between chunks. """ + if max_variants is None: + max_variants = constants.MAX_SOMATIC_VARIANTS + elif max_variants <= 0: + raise ValueError("max_variants must be a positive integer.") + output_dir = pathlib.Path(output_dir / "vcf_chunks") output_dir.mkdir(parents=True, exist_ok=True) chunk_files = [] @@ -547,7 +552,7 @@ def split_vcf(input_vcf, output_dir): for record in vcf_in: current_position = record.POS # Check if we need to start a new chunk - if variant_count >= constants.MAX_SOMATIC_VARIANTS and (last_position is None or current_position != last_position): + if variant_count >= max_variants and (last_position is None or current_position != last_position): # Close the current chunk file and start a new one vcf_out.close() chunk_number += 1 diff --git a/bolt/workflows/smlv_somatic/annotate.py b/bolt/workflows/smlv_somatic/annotate.py index d0efa93..32b85b3 100644 --- a/bolt/workflows/smlv_somatic/annotate.py +++ b/bolt/workflows/smlv_somatic/annotate.py @@ -30,6 +30,7 @@ @click.option('--threads', required=False, default=4, type=int) @click.option('--output_dir', required=True, type=click.Path()) +@click.option('--pcgr_variant_chunk_size', required=False, type=int, help='Override maximum variants per PCGR chunk.') def entry(ctx, **kwargs): '''Annotate variants with information from several sources\f @@ -99,8 +100,13 @@ def entry(ctx, **kwargs): print(f"Total number of variants in the input VCF: {total_variants}") # Run PCGR in chunks if exceeding the maximum allowed for somatic variants - if total_variants > constants.MAX_SOMATIC_VARIANTS: - vcf_chunks = pcgr.split_vcf(pcgr_prep_fp, output_dir) + chunk_size = kwargs.get('pcgr_variant_chunk_size') + if chunk_size is not None and chunk_size <= 0: + raise click.BadParameter('must be a positive integer', param_hint='--pcgr_variant_chunk_size') + chunk_size = chunk_size or constants.MAX_SOMATIC_VARIANTS + + if total_variants > chunk_size: + vcf_chunks = pcgr.split_vcf(pcgr_prep_fp, output_dir, max_variants=chunk_size) pcgr_tsv_fp, pcgr_vcf_fp = pcgr.run_somatic_chunck( vcf_chunks, kwargs['pcgr_data_dir'], From 8223ef9a729119bdbfefe7a16b3b7b4b25439200 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 17 Oct 2025 11:44:28 +1100 Subject: [PATCH 09/89] fix version software pcgr container --- docker/Dockerfile.pcgr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.pcgr b/docker/Dockerfile.pcgr index 8615e50..ca984e8 100644 --- a/docker/Dockerfile.pcgr +++ b/docker/Dockerfile.pcgr @@ -39,7 +39,7 @@ RUN \ RUN \ conda clean -afy -FROM quay.io/bioconda/base-glibc-busybox-bash:2.2.5 +FROM quay.io/bioconda/base-glibc-busybox-bash:2.1.0 # Copy Conda install and all environments COPY --from=build /opt/conda/ /opt/conda/ From f83c23aec1273df1fa9ddeaab354c279b3375352 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 17 Oct 2025 15:50:20 +1100 Subject: [PATCH 10/89] add logging and filter fonction to read value and replace placeholder --- bolt/workflows/smlv_somatic/filter.py | 59 +++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 8 deletions(-) diff --git a/bolt/workflows/smlv_somatic/filter.py b/bolt/workflows/smlv_somatic/filter.py index 814863c..bb5e8f1 100644 --- a/bolt/workflows/smlv_somatic/filter.py +++ b/bolt/workflows/smlv_somatic/filter.py @@ -1,12 +1,14 @@ import click import pathlib - +import logging import cyvcf2 from ... import util from ...common import constants +from ...logging_config import setup_logging +logger = logging.getLogger(__name__) @click.command(name='filter') @@ -129,7 +131,11 @@ def set_filter_data(record, tumor_index): # PON filter ## # NOTE(SW): 'max' is inclusive - keeps variants with 0 to n-1 PON hits; preserved from Umccrise - pon_count = record.INFO.get(constants.VcfInfo.PON_COUNT.value, 0) + pon_count = get_record_value( + record, + constants.VcfInfo.PON_COUNT.value, + default=0 + ) if pon_count >= constants.PON_HIT_THRESHOLD: filters.append(constants.VcfFilter.PON) @@ -144,7 +150,14 @@ def set_filter_data(record, tumor_index): ## # NOTE(SW): rounding is essential here for accurate comparison; cyvcf2 floating-point error # means INFO/gnomAD_AF=0.01 can be represented as 0.009999999776482582 - gnomad_af = round(record.INFO.get(constants.VcfInfo.GNOMAD_AF.value, 0), 3) + gnomad_af = round( + get_record_value( + record, + constants.VcfInfo.GNOMAD_AF.value, + default=0.0 ), + ), + 3, + ) if gnomad_af >= constants.MAX_GNOMAD_AF: filters.append(constants.VcfFilter.GNOMAD_COMMON) @@ -173,7 +186,7 @@ def set_filter_data(record, tumor_index): # SAGE hotspot rescue ## # NOTE(SW): effectively reverts any FILTERs that may have been applied above - if record.INFO.get(constants.VcfInfo.SAGE_HOTSPOT.value) is not None: + if get_record_value(record, constants.VcfInfo.SAGE_HOTSPOT.value) is not None: info_rescue.append(constants.VcfInfo.SAGE_HOTSPOT_RESCUE) ## @@ -184,13 +197,24 @@ def set_filter_data(record, tumor_index): # single CLINICAL_POTENTIAL_RESCUE flag # Get ClinVar clinical significance entries - clinvar_clinsig = record.INFO.get(constants.VcfInfo.PCGR_CLINVAR_CLASSIFICATION.value, '') + clinvar_clinsig = get_record_value( + record, + constants.VcfInfo.PCGR_CLINVAR_CLASSIFICATION.value, + default='', + ) clinvar_clinsigs = clinvar_clinsig.split(',') # Hit counts in relevant reference somatic mutation databases - tcga_pancancer_count = record.INFO.get(constants.VcfInfo.PCGR_TCGA_PANCANCER_COUNT.value, 0) + tcga_pancancer_count = get_record_value( + record, + constants.VcfInfo.PCGR_TCGA_PANCANCER_COUNT.value, + default=0 + ) + hmf_present = get_record_value(record, constants.VcfInfo.HMF_HOTSPOT.value) + pcgr_hotspot_present = get_record_value(record, constants.VcfInfo.PCGR_MUTATION_HOTSPOT.value) + if ( - record.INFO.get(constants.VcfInfo.HMF_HOTSPOT.value) is not None or - record.INFO.get(constants.VcfInfo.PCGR_MUTATION_HOTSPOT.value) is not None or + hmf_present is not None or + pcgr_hotspot_present is not None or any(e in clinvar_clinsigs for e in constants.CLINVAR_CLINSIGS_RESCUE) or tcga_pancancer_count >= constants.MIN_TCGA_PANCANCER_COUNT_RESCUE ): @@ -227,3 +251,22 @@ def set_filter_data(record, tumor_index): filters_existing = [e for e in record.FILTERS if e != 'PASS'] assert all(e not in filters_existing for e in filters_value) record.FILTER = ';'.join([*filters_existing, *filters_value]) + + +def get_record_value(record, key, *, default=None): + ''' + Return a INFO value, replacing '.', '' and missing entries by default value. + + This prevents placeholder values emitted by pcgr from triggering rescues + or filters. For example, an INFO placeholder '.' for + PCGR_TCGA_PANCANCER_COUNT will resolve to the integer 0 when called with + `get_record_value(record, 'PCGR_TCGA_PANCANCER_COUNT', default=0)`. + ''' + value = record.INFO.get(key) + if value is None: + return default + if value=='' or value=='.': + return default + if not isinstance(value, (int, float, str)): + logger.error(f'record ID: {record.ID} INFO value for {key}: {value} ({type(value)}, not int, float, str)') + return value \ No newline at end of file From 22b5c1291b7d7432a0c65c7ff5c75bdf07cd71ce Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 17 Oct 2025 16:02:38 +1100 Subject: [PATCH 11/89] fix indentation --- bolt/workflows/smlv_somatic/filter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bolt/workflows/smlv_somatic/filter.py b/bolt/workflows/smlv_somatic/filter.py index bb5e8f1..c820385 100644 --- a/bolt/workflows/smlv_somatic/filter.py +++ b/bolt/workflows/smlv_somatic/filter.py @@ -154,7 +154,7 @@ def set_filter_data(record, tumor_index): get_record_value( record, constants.VcfInfo.GNOMAD_AF.value, - default=0.0 ), + default=0.0 ), 3, ) @@ -214,7 +214,7 @@ def set_filter_data(record, tumor_index): if ( hmf_present is not None or - pcgr_hotspot_present is not None or + pcgr_hotspot_present or any(e in clinvar_clinsigs for e in constants.CLINVAR_CLINSIGS_RESCUE) or tcga_pancancer_count >= constants.MIN_TCGA_PANCANCER_COUNT_RESCUE ): From 529309c03d4838f8c5fe71c704dfee298a932933 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 17 Oct 2025 16:36:13 +1100 Subject: [PATCH 12/89] read gz tsv from pcgr --- bolt/common/pcgr.py | 2 +- bolt/util.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bolt/common/pcgr.py b/bolt/common/pcgr.py index 54c87aa..a45154a 100644 --- a/bolt/common/pcgr.py +++ b/bolt/common/pcgr.py @@ -604,7 +604,7 @@ def merging_pcgr_files(output_dir, pcgr_vcf_files, pcgr_tsv_fp): pcgr_dir.mkdir(exist_ok=True) # Merge all TSV files into a single file in the pcgr directory - merged_tsv_fp = pcgr_dir / "nosampleset.pcgr_acmg.grch38.snvs_indels.tiers.tsv" + merged_tsv_fp = pcgr_dir / "nosampleset.pcgr_acmg.grch38.snvs_indels.tiers.tsv.gz" util.merge_tsv_files(pcgr_tsv_fp, merged_tsv_fp) # Step 5: Merge all VCF files into a single file in the pcgr directory diff --git a/bolt/util.py b/bolt/util.py index 7310b89..8065302 100644 --- a/bolt/util.py +++ b/bolt/util.py @@ -1,3 +1,4 @@ +import gzip import pathlib import subprocess import textwrap @@ -161,11 +162,12 @@ def split_vcf(input_vcf, output_dir): def merge_tsv_files(tsv_files, merged_tsv_fp): """ - Merges all TSV files into a single TSV. + Merge gzipped TSV files into a single gzipped TSV. """ - with open(merged_tsv_fp, 'w') as merged_tsv: + + with gzip.open(merged_tsv_fp, 'wt', encoding='utf-8') as merged_tsv: for i, tsv_file in enumerate(tsv_files): - with open(tsv_file, 'r') as infile: + with gzip.open(tsv_file, 'rt', encoding='utf-8') as infile: for line_number, line in enumerate(infile): # Skip header except for the first file if i > 0 and line_number == 0: From d3dd1820ba346e513e79f4427dade621baca0184 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Mon, 20 Oct 2025 14:54:57 +1100 Subject: [PATCH 13/89] change PCGR_CLINVAR_CLASSIFICATION --- bolt/common/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bolt/common/constants.py b/bolt/common/constants.py index 7cebe78..cfc6aa5 100644 --- a/bolt/common/constants.py +++ b/bolt/common/constants.py @@ -301,7 +301,7 @@ def namespace(self): VcfInfo.PCGR_CLINVAR_CLASSIFICATION: { 'Number': '.', 'Type': 'String', - 'Description': 'ClinVar - clinical significance - per phenotype submission', + 'Description': 'ClinVar - Overall clinical significance of variant on a five-tiered scale', }, VcfInfo.PCGR_COSMIC_COUNT: { 'Number': '1', From a5fc92311f4607d0d528151ca83394c150f74b68 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Mon, 20 Oct 2025 15:00:46 +1100 Subject: [PATCH 14/89] Change pcgr header check --- bolt/common/pcgr.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/bolt/common/pcgr.py b/bolt/common/pcgr.py index a45154a..9dba668 100644 --- a/bolt/common/pcgr.py +++ b/bolt/common/pcgr.py @@ -348,6 +348,7 @@ def check_annotation_headers(info_field_map, vcf_fp): # Ensure header descriptions from source INFO annotations match those defined here for the # output file; force manual inspection where they do not match vcf_fh = cyvcf2.VCF(vcf_fp) + mismatches = list() for header_dst, header_src in info_field_map.items(): # Skip header lines that do not have an equivalent entry in the PCGR/CPSR VCF @@ -361,7 +362,15 @@ def check_annotation_headers(info_field_map, vcf_fp): # Remove leading and trailing quotes from source header_src_description_unquoted = header_src_entry['Description'].strip('"') if header_src_description_unquoted != header_dst_entry['Description']: - raise AssertionError(f"Mismatch for {header_src}:\nVCF: {header_src_description_unquoted}\nExpected: {header_dst_entry['Description']}") + mismatches.append( + f"Mismatch for {header_src}:\n" + f"VCF: {header_src_description_unquoted}\n" + f"Expected: {header_dst_entry['Description']}" + ) + + if mismatches: + mismatch_msg = "\n\n".join(mismatches) + raise AssertionError(f"Mismatched INFO annotations:\n{mismatch_msg}") def collect_pcgr_annotation_data(tsv_fp, vcf_fp, info_field_map): # Gather all annotations from TSV @@ -608,7 +617,7 @@ def merging_pcgr_files(output_dir, pcgr_vcf_files, pcgr_tsv_fp): util.merge_tsv_files(pcgr_tsv_fp, merged_tsv_fp) # Step 5: Merge all VCF files into a single file in the pcgr directory - merged_vcf_path = pcgr_dir / "nosampleset.pcgr.grch38.pass.vcf.gz" + merged_vcf_path = pcgr_dir / "nosampleset.pcgr.grch38.pass" merged_vcf = util.merge_vcf_files(pcgr_vcf_files, merged_vcf_path) return merged_vcf, merged_tsv_fp From bf930e78188c403f530ed754f1f9b451b70e0691 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 24 Oct 2025 13:56:03 +1100 Subject: [PATCH 15/89] tyo --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90b7815..25ce706 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,4 +10,4 @@ - [6](https://github.com/umccr/bolt/pull/6) - Change oncoanalyser v2.0.0 uptade, with switch sv caller from GRIPSS to eSVee --[9](https://github.com/umccr/bolt/pull/9) Add hypermutation sample handling \ No newline at end of file +- [9](https://github.com/umccr/bolt/pull/9) Add hypermutation sample handling \ No newline at end of file From 8ea1edc8cd5c64afff30640471f515f279b2c90a Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 24 Oct 2025 13:57:30 +1100 Subject: [PATCH 16/89] typo --- bolt/common/pcgr.py | 2 +- bolt/workflows/other/cancer_report.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bolt/common/pcgr.py b/bolt/common/pcgr.py index 9dba668..95c0f47 100644 --- a/bolt/common/pcgr.py +++ b/bolt/common/pcgr.py @@ -649,7 +649,7 @@ def get_variant_filter_data(variant): # NOTE(SW): GIAB_CONF always overrides DIFFICULT tags if data['giab_conf'] and data['difficult']: - data['difficult']= False + data['difficult'] = False for impact in get_impacts(variant.INFO['PCGR_CSQ']): diff --git a/bolt/workflows/other/cancer_report.py b/bolt/workflows/other/cancer_report.py index 1902079..9ad3d93 100644 --- a/bolt/workflows/other/cancer_report.py +++ b/bolt/workflows/other/cancer_report.py @@ -69,7 +69,7 @@ def entry(ctx, **kwargs): ) # Set other required argument values - batch_name = f'{kwargs["subject_name"]}_{kwargs["tumor_name"]}' + batch_name = f'{kwargs['subject_name']}_{kwargs['tumor_name']}' output_table_dir = output_dir / 'cancer_report_tables' # Optional dragen hrd argument From 443c5be931e062b00eb56a74e7bb3221c87a4bdc Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 24 Oct 2025 13:57:56 +1100 Subject: [PATCH 17/89] add missing variable pcgr_output_dir --- bolt/workflows/smlv_somatic/report.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bolt/workflows/smlv_somatic/report.py b/bolt/workflows/smlv_somatic/report.py index ede90ca..65ce535 100644 --- a/bolt/workflows/smlv_somatic/report.py +++ b/bolt/workflows/smlv_somatic/report.py @@ -129,6 +129,7 @@ def entry(ctx, **kwargs): output_dir, ) + pcgr_output_dir = output_dir / 'pcgr' pcgr.run_somatic( pcgr_prep_fp, kwargs['pcgr_data_dir'], From 966967fb9e2003a450941fa872d44d1b95543d18 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen <37511834+qclayssen@users.noreply.github.com> Date: Wed, 19 Nov 2025 14:25:55 +1100 Subject: [PATCH 18/89] Pcgr v2.2.5 (#22) * update "HIGH_INF_POS" to "HIGH_I_I_POS" in VCF header for latest pcgr version * update constant for pcgr 2.2.5 --- bolt/common/constants.py | 17 +++++------------ bolt/workflows/other/cancer_report.py | 1 + bolt/workflows/smlv_germline/report.py | 3 +++ bolt/workflows/smlv_somatic/report.py | 3 +++ docker/Dockerfile.pcgr | 4 ++-- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/bolt/common/constants.py b/bolt/common/constants.py index cfc6aa5..bc189ae 100644 --- a/bolt/common/constants.py +++ b/bolt/common/constants.py @@ -288,8 +288,10 @@ def namespace(self): 'MANE|MANE_SELECT|MANE_PLUS_CLINICAL|TSL|APPRIS|CCDS|ENSP|SWISSPROT|TREMBL|UNIPARC|' 'UNIPROT_ISOFORM|RefSeq|DOMAINS|HGVS_OFFSET|gnomADe_AF|gnomADe_AFR_AF|gnomADe_AMR_AF|' 'gnomADe_ASJ_AF|gnomADe_EAS_AF|gnomADe_FIN_AF|gnomADe_MID_AF|gnomADe_NFE_AF|' - 'gnomADe_REMAINING_AF|gnomADe_SAS_AF|CLIN_SIG|SOMATIC|PHENO|CHECK_REF|MOTIF_NAME|' - 'MOTIF_POS|HIGH_INF_POS|MOTIF_SCORE_CHANGE|TRANSCRIPTION_FACTORS|NearestExonJB|' + 'gnomADe_REMAINING_AF|gnomADe_SAS_AF|gnomADg_AF|gnomADg_AFR_AF|gnomADg_AMI_AF|' + 'gnomADg_AMR_AF|gnomADg_ASJ_AF|gnomADg_EAS_AF|gnomADg_FIN_AF|gnomADg_MID_AF|' + 'gnomADg_NFE_AF|gnomADg_REMAINING_AF|gnomADg_SAS_AF|CLIN_SIG|SOMATIC|PHENO|CHECK_REF|' + 'MOTIF_NAME|MOTIF_POS|HIGH_INF_POS|MOTIF_SCORE_CHANGE|TRANSCRIPTION_FACTORS|NearestExonJB|' 'MaxEntScan_alt|MaxEntScan_diff|MaxEntScan_ref' ), }, @@ -328,16 +330,7 @@ def namespace(self): 'Number': '.', 'Type': 'String', 'Description': ( - 'Consequence annotations from Ensembl VEP. Format: Allele|Consequence|IMPACT|SYMBOL|' - 'Gene|Feature_type|Feature|BIOTYPE|EXON|INTRON|HGVSc|HGVSp|cDNA_position|CDS_position|' - 'Protein_position|Amino_acids|Codons|Existing_variation|ALLELE_NUM|DISTANCE|STRAND|' - 'FLAGS|PICK|VARIANT_CLASS|SYMBOL_SOURCE|HGNC_ID|CANONICAL|MANE|MANE_SELECT|' - 'MANE_PLUS_CLINICAL|TSL|APPRIS|CCDS|ENSP|SWISSPROT|TREMBL|UNIPARC|UNIPROT_ISOFORM|RefSeq|' - 'DOMAINS|HGVS_OFFSET|gnomADe_AF|gnomADe_AFR_AF|gnomADe_AMR_AF|gnomADe_ASJ_AF|' - 'gnomADe_EAS_AF|gnomADe_FIN_AF|gnomADe_MID_AF|gnomADe_NFE_AF|gnomADe_REMAINING_AF|' - 'gnomADe_SAS_AF|CLIN_SIG|SOMATIC|PHENO|CHECK_REF|MOTIF_NAME|MOTIF_POS|HIGH_INF_POS|' - 'MOTIF_SCORE_CHANGE|TRANSCRIPTION_FACTORS|NearestExonJB|MaxEntScan_alt|MaxEntScan_diff|' - 'MaxEntScan_ref' + 'Consequence annotations from Ensembl VEP. Format: Allele|Consequence|IMPACT|SYMBOL|Gene|Feature_type|Feature|BIOTYPE|EXON|INTRON|HGVSc|HGVSp|cDNA_position|CDS_position|Protein_position|Amino_acids|Codons|Existing_variation|ALLELE_NUM|DISTANCE|STRAND|FLAGS|PICK|VARIANT_CLASS|SYMBOL_SOURCE|HGNC_ID|CANONICAL|MANE|MANE_SELECT|MANE_PLUS_CLINICAL|TSL|APPRIS|CCDS|ENSP|SWISSPROT|TREMBL|UNIPARC|UNIPROT_ISOFORM|RefSeq|DOMAINS|HGVS_OFFSET|gnomADe_AF|gnomADe_AFR_AF|gnomADe_AMR_AF|gnomADe_ASJ_AF|gnomADe_EAS_AF|gnomADe_FIN_AF|gnomADe_MID_AF|gnomADe_NFE_AF|gnomADe_REMAINING_AF|gnomADe_SAS_AF|gnomADg_AF|gnomADg_AFR_AF|gnomADg_AMI_AF|gnomADg_AMR_AF|gnomADg_ASJ_AF|gnomADg_EAS_AF|gnomADg_FIN_AF|gnomADg_MID_AF|gnomADg_NFE_AF|gnomADg_REMAINING_AF|gnomADg_SAS_AF|CLIN_SIG|SOMATIC|PHENO|CHECK_REF|MOTIF_NAME|MOTIF_POS|HIGH_INF_POS|MOTIF_SCORE_CHANGE|TRANSCRIPTION_FACTORS|NearestExonJB|MaxEntScan_alt|MaxEntScan_diff|MaxEntScan_ref' ), }, diff --git a/bolt/workflows/other/cancer_report.py b/bolt/workflows/other/cancer_report.py index 9ad3d93..330e095 100644 --- a/bolt/workflows/other/cancer_report.py +++ b/bolt/workflows/other/cancer_report.py @@ -2,6 +2,7 @@ import click +import logging from ... import util diff --git a/bolt/workflows/smlv_germline/report.py b/bolt/workflows/smlv_germline/report.py index 395268f..224907b 100644 --- a/bolt/workflows/smlv_germline/report.py +++ b/bolt/workflows/smlv_germline/report.py @@ -4,11 +4,14 @@ import click +import logging from ... import util from ...common import pcgr +logger = logging.getLogger(__name__) + @click.command(name='report') @click.pass_context diff --git a/bolt/workflows/smlv_somatic/report.py b/bolt/workflows/smlv_somatic/report.py index 65ce535..0954cc3 100644 --- a/bolt/workflows/smlv_somatic/report.py +++ b/bolt/workflows/smlv_somatic/report.py @@ -7,11 +7,14 @@ import click import cyvcf2 import yaml +import logging from ... import util from ...common import constants from ...common import pcgr +from ...logging_config import setup_logging +logger = logging.getLogger(__name__) @click.command(name='report') diff --git a/docker/Dockerfile.pcgr b/docker/Dockerfile.pcgr index ca984e8..2619305 100644 --- a/docker/Dockerfile.pcgr +++ b/docker/Dockerfile.pcgr @@ -15,13 +15,13 @@ RUN \ conda create \ --solver libmamba \ --name pcgr \ - --file https://raw.githubusercontent.com/sigven/pcgr/refs/tags/v2.2.1/conda/env/lock/pcgr-linux-64.lock + --file https://raw.githubusercontent.com/sigven/pcgr/refs/tags/v2.2.5/conda/env/lock/pcgr-linux-64.lock RUN \ conda create \ --solver libmamba \ --name pcgrr \ - --file https://raw.githubusercontent.com/sigven/pcgr/refs/tags/v2.2.1/conda/env/lock/pcgrr-linux-64.lock + --file https://raw.githubusercontent.com/sigven/pcgr/refs/tags/v2.2.5/conda/env/lock/pcgrr-linux-64.lock COPY ./conda/env/bolt_env.yml /tmp/ RUN \ From 6a93f6b79d317501c36fb7ac393748c750725e65 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen <37511834+qclayssen@users.noreply.github.com> Date: Mon, 24 Nov 2025 15:06:20 +1100 Subject: [PATCH 19/89] Uptade sage vcf header check (#24) * change sage vcf header to match 2024 version * move check function * add check in rescue for sage vcf * change for consistencency with pcgr --- bolt/common/constants.py | 8 +++---- bolt/common/pcgr.py | 32 ++------------------------- bolt/util.py | 23 +++++++++++++++++++ bolt/workflows/smlv_somatic/rescue.py | 21 +++++++++++++----- 4 files changed, 43 insertions(+), 41 deletions(-) diff --git a/bolt/common/constants.py b/bolt/common/constants.py index bc189ae..6bbc610 100644 --- a/bolt/common/constants.py +++ b/bolt/common/constants.py @@ -368,7 +368,7 @@ def namespace(self): }, VcfInfo.RESCUED_FILTERS_PENDING: { - 'Number': '1', + 'Number': '.', 'Type': 'String', 'Description': 'Filters pending prior to variant rescue', }, @@ -397,13 +397,11 @@ def namespace(self): 'Description': 'Approximate read depth (reads with MQ=255 or with bad mates are filtered)', }, VcfFormat.SAGE_SB: { - 'Number': '1', + 'Number': '2', 'Type': 'Float', - 'Description': 'Strand bias - percentage of first-in-pair reads', + 'Description': 'Fragment strand bias - percentage of forward-orientation fragments (ref,alt)', }, } - - ##################### ## Other ## ##################### diff --git a/bolt/common/pcgr.py b/bolt/common/pcgr.py index 95c0f47..3690f79 100644 --- a/bolt/common/pcgr.py +++ b/bolt/common/pcgr.py @@ -280,7 +280,7 @@ def transfer_annotations_somatic(input_fp, tumor_name, pcgr_vcf_fp, pcgr_tsv_fp, pcgr_vcf_fp = pathlib.Path(pcgr_vcf_fp) # Enforce matching defined and source INFO annotations - check_annotation_headers(info_field_map, pcgr_vcf_fp) + util.check_annotation_headers(info_field_map, pcgr_vcf_fp) # Gather PCGR annotation data for records pcgr_data = collect_pcgr_annotation_data(pcgr_tsv_fp, pcgr_vcf_fp, info_field_map) @@ -319,7 +319,7 @@ def transfer_annotations_germline(input_fp, normal_name, cpsr_dir, output_dir): cpsr_vcf_fp = pathlib.Path(cpsr_dir) / f'{normal_name}.cpsr.grch38.vcf.gz' # Enforce matching defined and source INFO annotations - check_annotation_headers(info_field_map, cpsr_vcf_fp) + util.check_annotation_headers(info_field_map, cpsr_vcf_fp) # Gather CPSR annotation data for records cpsr_data = collect_cpsr_annotation_data(cpsr_tsv_fp, cpsr_vcf_fp, info_field_map) @@ -344,34 +344,6 @@ def transfer_annotations_germline(input_fp, normal_name, cpsr_dir, output_dir): output_fh.write_record(record_ann) -def check_annotation_headers(info_field_map, vcf_fp): - # Ensure header descriptions from source INFO annotations match those defined here for the - # output file; force manual inspection where they do not match - vcf_fh = cyvcf2.VCF(vcf_fp) - mismatches = list() - for header_dst, header_src in info_field_map.items(): - - # Skip header lines that do not have an equivalent entry in the PCGR/CPSR VCF - try: - header_src_entry = vcf_fh.get_header_type(header_src) - except KeyError: - print(f"Missing in VCF: {header_src}") - continue - - header_dst_entry = util.get_vcf_header_entry(header_dst) - # Remove leading and trailing quotes from source - header_src_description_unquoted = header_src_entry['Description'].strip('"') - if header_src_description_unquoted != header_dst_entry['Description']: - mismatches.append( - f"Mismatch for {header_src}:\n" - f"VCF: {header_src_description_unquoted}\n" - f"Expected: {header_dst_entry['Description']}" - ) - - if mismatches: - mismatch_msg = "\n\n".join(mismatches) - raise AssertionError(f"Mismatched INFO annotations:\n{mismatch_msg}") - def collect_pcgr_annotation_data(tsv_fp, vcf_fp, info_field_map): # Gather all annotations from TSV data_tsv = dict() diff --git a/bolt/util.py b/bolt/util.py index 8065302..4c427df 100644 --- a/bolt/util.py +++ b/bolt/util.py @@ -1,6 +1,7 @@ import gzip import pathlib import subprocess +import sys import textwrap import logging from types import SimpleNamespace @@ -249,3 +250,25 @@ def merge_vcf_files(vcf_files, merged_vcf_fp): merged_unsorted_vcf.unlink() return merged_vcf + +def check_annotation_headers(info_field_map, vcf_fp): + # Ensure header descriptions from source INFO annotations match those defined here for the + # output file; force manual inspection where they do not match + vcf_fh = cyvcf2.VCF(vcf_fp) + for header_dst, header_src in info_field_map.items(): + # Skip header lines that do not have an equivalent entry in the VCF + try: + header_src_entry = vcf_fh.get_header_type(header_src) + except KeyError: + continue + + header_dst_entry = get_vcf_header_entry(header_dst) + # Remove leading and trailing quotes from source + header_src_description_unquoted = header_src_entry['Description'].strip('"') + try: + assert header_src_description_unquoted == header_dst_entry['Description'] + except AssertionError: + print(f'Header description mismatch for {header_dst.value}') + print(f' src: {header_src_description_unquoted}') + print(f' dst: {header_dst_entry["Description"]}') + sys.exit(1) diff --git a/bolt/workflows/smlv_somatic/rescue.py b/bolt/workflows/smlv_somatic/rescue.py index bfd89b9..4a9973b 100644 --- a/bolt/workflows/smlv_somatic/rescue.py +++ b/bolt/workflows/smlv_somatic/rescue.py @@ -113,20 +113,29 @@ def annotate_existing_sage_calls(input_fp, tumor_name, sage_vcf_fp, output_dir): # Get input file handle input_fh = cyvcf2.VCF(input_fp) + # Get input file handle + # Perform consistency check on the VCF header + info_field_map_sage = { + constants.VcfInfo.SAGE_HOTSPOT: 'SAGE_HOTSPOT', + constants.VcfInfo.SAGE_NOVEL: 'SAGE_NOVEL', + constants.VcfInfo.SAGE_RESCUE: 'SAGE_RESCUE', + constants.VcfFormat.SAGE_AD: 'SAGE_AD', + constants.VcfFormat.SAGE_AF: 'SAGE_AF', + constants.VcfFormat.SAGE_DP: 'SAGE_DP', + constants.VcfFormat.SAGE_SB: 'SAGE_SB', + constants.VcfFilter.SAGE_LOWCONF: 'SAGE_LOWCONF', + } + + util.check_annotation_headers(info_field_map_sage, input_fp) + # Add header entries so that they are included in the output file via templating done below util.add_vcf_header_entry(input_fh, constants.VcfFilter.SAGE_LOWCONF) - util.add_vcf_header_entry(input_fh, constants.VcfInfo.SAGE_HOTSPOT) util.add_vcf_header_entry(input_fh, constants.VcfInfo.SAGE_RESCUE) - - # TODO(SW): check that defined header descriptions match those in the SAGE fp; collect as list - # here and iterate to check and then add to input_fp header also in another loop - util.add_vcf_header_entry(input_fh, constants.VcfFormat.SAGE_AD) util.add_vcf_header_entry(input_fh, constants.VcfFormat.SAGE_AF) util.add_vcf_header_entry(input_fh, constants.VcfFormat.SAGE_DP) util.add_vcf_header_entry(input_fh, constants.VcfFormat.SAGE_SB) - # Open output file and use header from input file output_fp = output_dir / f'{tumor_name}.anno.vcf.gz' output_fh = cyvcf2.Writer(output_fp, input_fh, 'wz') From ed3b11b3f2c8b31fed683ff87a8e69a1e210f434 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Tue, 25 Nov 2025 10:18:20 +1100 Subject: [PATCH 20/89] add missing HMF_HOTSPOT constant --- bolt/common/constants.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bolt/common/constants.py b/bolt/common/constants.py index 6bbc610..a7a207d 100644 --- a/bolt/common/constants.py +++ b/bolt/common/constants.py @@ -315,6 +315,11 @@ def namespace(self): 'Type': 'Integer', 'Description': 'Raw variant count across all tumor types', }, + VcfInfo.HMF_HOTSPOT: { + 'Number': '0', + 'Type': 'Flag', + 'Description': 'calculated by flag of overlapping values in field HMF from annotations/hotspots/hotspots.hmf.vcf.gz', + }, VcfInfo.PCGR_ICGC_PCAWG_COUNT: { 'Number': '1', 'Type': 'Integer', From a3d16f7fbe1addaaa1b6a861a30fcf68f6e85d11 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Tue, 25 Nov 2025 10:21:30 +1100 Subject: [PATCH 21/89] Fix quotation --- bolt/workflows/other/cancer_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bolt/workflows/other/cancer_report.py b/bolt/workflows/other/cancer_report.py index 330e095..7e2d70f 100644 --- a/bolt/workflows/other/cancer_report.py +++ b/bolt/workflows/other/cancer_report.py @@ -70,7 +70,7 @@ def entry(ctx, **kwargs): ) # Set other required argument values - batch_name = f'{kwargs['subject_name']}_{kwargs['tumor_name']}' + batch_name = f"{kwargs['subject_name']}_{kwargs['tumor_name']}" output_table_dir = output_dir / 'cancer_report_tables' # Optional dragen hrd argument From 9ddeac8c259da5ce6d06072103d4deefde508726 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Tue, 25 Nov 2025 10:21:48 +1100 Subject: [PATCH 22/89] Fix test --- tests/test_smlv_somatic_filter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_smlv_somatic_filter.py b/tests/test_smlv_somatic_filter.py index 6455162..f872227 100644 --- a/tests/test_smlv_somatic_filter.py +++ b/tests/test_smlv_somatic_filter.py @@ -46,10 +46,10 @@ def get_record_from_str(variant_str): def get_record( chrom='chr1', - pos='.', + pos='1', vid='.', - ref='.', - alt='.', + ref='A', + alt='T', qual='.', vfilter='.', info_data=None, From 41b83bb3b9c328c88396d8021f8852a6f2ceca19 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Tue, 16 Dec 2025 16:31:42 +1100 Subject: [PATCH 23/89] remove PCGR html report from chunk (hypermutated) to reduce memory use --- bolt/common/pcgr.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bolt/common/pcgr.py b/bolt/common/pcgr.py index 3690f79..b6328e1 100644 --- a/bolt/common/pcgr.py +++ b/bolt/common/pcgr.py @@ -173,6 +173,9 @@ def run_somatic(input_fp, pcgr_refdata_dir, vep_dir, output_dir, chunk_nbr=None, if ploidy: command_args.append(f'--tumor_ploidy {ploidy}') + if chunk_nbr is not None: + command_args.append(f'--no_html') + # NOTE(SW): placed here to always have output directory last command_args.append(f'--output_dir {output_dir}') From c42d725e48e12a173e7d09820c803f4f76478f00 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Wed, 17 Dec 2025 15:24:55 +1100 Subject: [PATCH 24/89] change remove pcgr parallelisation of chunks, and fix pcgr internal parallelisation because OOM hypermutated samples --- bolt/common/pcgr.py | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/bolt/common/pcgr.py b/bolt/common/pcgr.py index b6328e1..e881fb3 100644 --- a/bolt/common/pcgr.py +++ b/bolt/common/pcgr.py @@ -557,29 +557,15 @@ def split_vcf(input_vcf, output_dir, *, max_variants=None): def run_somatic_chunck(vcf_chunks, pcgr_data_dir, vep_dir, output_dir, pcgr_output_dir, max_threads, pcgr_conda, pcgrr_conda): pcgr_tsv_files = [] pcgr_vcf_files = [] - num_chunks = len(vcf_chunks) - # Ensure we don't use more workers than available threads, and each worker has at least 2 threads - max_workers = min(num_chunks, max_threads // 2) - threads_quot, threads_rem = divmod(max_threads, num_chunks) - threads_per_chunk = max(2, threads_quot) - # Limit the number of workers to the smaller of num_chunks or max_threads // 2 - with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor: - futures = {} - for chunk_number, vcf_file in enumerate(vcf_chunks, start=1): - # Assign extra thread to the first 'threads_rem' chunks - additional_thread = 1 if chunk_number <= threads_rem else 0 - total_threads = threads_per_chunk + additional_thread - futures[executor.submit(run_somatic, vcf_file, pcgr_data_dir, vep_dir, pcgr_output_dir, chunk_number, total_threads, pcgr_conda, pcgrr_conda)] = chunk_number - for future in concurrent.futures.as_completed(futures): - try: - pcgr_tsv_fp, pcgr_vcf_fp = future.result() - if pcgr_tsv_fp: - pcgr_tsv_files.append(pcgr_tsv_fp) - if pcgr_vcf_fp: - pcgr_vcf_files.append(pcgr_vcf_fp) - except Exception: - chunk_number = futures[future] - logger.exception(f"Exception occurred while processing PCGR chunk {chunk_number}.") + for chunk_number, vcf_file in enumerate(vcf_chunks, start=1): + try: + pcgr_tsv_fp, pcgr_vcf_fp = run_somatic(vcf_file, pcgr_data_dir, vep_dir, pcgr_output_dir, chunk_number, max_threads, pcgr_conda, pcgrr_conda) + if pcgr_tsv_fp: + pcgr_tsv_files.append(pcgr_tsv_fp) + if pcgr_vcf_fp: + pcgr_vcf_files.append(pcgr_vcf_fp) + except Exception: + logger.exception(f"Exception occurred while processing PCGR chunk {chunk_number}.") merged_vcf_fp, merged_tsv_fp = merging_pcgr_files(output_dir, pcgr_vcf_files, pcgr_tsv_files) return merged_tsv_fp, merged_vcf_fp From 03bba1d8257b458228740a2c04ee3c3604a7b924 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Wed, 17 Dec 2025 16:00:24 +1100 Subject: [PATCH 25/89] free disk worker --- .github/workflows/build.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index cd8796a..8e40ae2 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -17,6 +17,21 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: false + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: From cb1c39b915e64ab851ff1657cc7b706e37f2a0c0 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 19 Dec 2025 14:56:52 +1100 Subject: [PATCH 26/89] remove exception to stop at fail process --- bolt/common/pcgr.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/bolt/common/pcgr.py b/bolt/common/pcgr.py index e881fb3..255ce8a 100644 --- a/bolt/common/pcgr.py +++ b/bolt/common/pcgr.py @@ -557,15 +557,15 @@ def split_vcf(input_vcf, output_dir, *, max_variants=None): def run_somatic_chunck(vcf_chunks, pcgr_data_dir, vep_dir, output_dir, pcgr_output_dir, max_threads, pcgr_conda, pcgrr_conda): pcgr_tsv_files = [] pcgr_vcf_files = [] + + # Process each chunk sequentially for chunk_number, vcf_file in enumerate(vcf_chunks, start=1): - try: - pcgr_tsv_fp, pcgr_vcf_fp = run_somatic(vcf_file, pcgr_data_dir, vep_dir, pcgr_output_dir, chunk_number, max_threads, pcgr_conda, pcgrr_conda) - if pcgr_tsv_fp: - pcgr_tsv_files.append(pcgr_tsv_fp) - if pcgr_vcf_fp: - pcgr_vcf_files.append(pcgr_vcf_fp) - except Exception: - logger.exception(f"Exception occurred while processing PCGR chunk {chunk_number}.") + pcgr_tsv_fp, pcgr_vcf_fp = run_somatic(vcf_file, pcgr_data_dir, vep_dir, pcgr_output_dir, chunk_number, max_threads, pcgr_conda, pcgrr_conda) + if pcgr_tsv_fp: + pcgr_tsv_files.append(pcgr_tsv_fp) + if pcgr_vcf_fp: + pcgr_vcf_files.append(pcgr_vcf_fp) + merged_vcf_fp, merged_tsv_fp = merging_pcgr_files(output_dir, pcgr_vcf_files, pcgr_tsv_files) return merged_tsv_fp, merged_vcf_fp From 27403cb1766e6d0355d2bb43153f4f4e04cd6208 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 19 Dec 2025 15:11:55 +1100 Subject: [PATCH 27/89] make thread you by pcgr constant, so in case of OOM, on the retry don't use more core and so more memory --- bolt/common/pcgr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bolt/common/pcgr.py b/bolt/common/pcgr.py index 255ce8a..ecbe6b1 100644 --- a/bolt/common/pcgr.py +++ b/bolt/common/pcgr.py @@ -146,7 +146,7 @@ def run_somatic(input_fp, pcgr_refdata_dir, vep_dir, output_dir, chunk_nbr=None, f'--estimate_signatures', f'--estimate_msi', f'--estimate_tmb', - f'--vcfanno_n_proc {threads}', + f'--vcfanno_n_proc 4', f'--vep_n_forks 4', f'--vep_pick_order biotype,rank,appris,tsl,ccds,canonical,length,mane_plus_clinical,mane_select', ] From 290242878bb8e5bce87d948dece585cf8091b362 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 19 Dec 2025 15:53:27 +1100 Subject: [PATCH 28/89] Fix silent command failures by integrating error handling into execute_command. Fixes #26 - Commands now fail immediately instead of continuing silently --- bolt/util.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/bolt/util.py b/bolt/util.py index 4c427df..726ffc9 100644 --- a/bolt/util.py +++ b/bolt/util.py @@ -22,6 +22,10 @@ def get_project_root(): def execute_command(command, log_file_path=None): + # Wrap command with proper error handling + # set -e: exit on error, -u: exit on unset variable, -o pipefail: pipeline fails if any command fails + prepared_command = f'set -euo pipefail; {textwrap.dedent(command)}' + logger.info("Executing command: %s", command.strip()) # Open the log file if provided @@ -29,7 +33,7 @@ def execute_command(command, log_file_path=None): # Launch process with combined stdout and stderr streams, and line buffering enabled. process = subprocess.Popen( - command, + prepared_command, shell=True, executable='/bin/bash', stdout=subprocess.PIPE, @@ -61,17 +65,16 @@ def execute_command(command, log_file_path=None): command=command ) - return result + # Raise exception on non-zero return code + if result.returncode != 0: + error_msg = f"Command failed with return code {result.returncode}: {command.strip()}" + logger.error(error_msg) + raise subprocess.CalledProcessError(result.returncode, command, output=''.join(output_lines)) -def command_prepare(command): - return f'set -o pipefail; {textwrap.dedent(command)}' + return result def count_vcf_records(fp): - result = subprocess.run(f'bcftools view -H {fp} | wc -l', - shell=True, - executable="/bin/bash", - capture_output=True, - text=True ) + result = execute_command(f'bcftools view -H {fp} | wc -l') return int(result.stdout.strip()) From 15db538c3dcb6ad3bc9f6716448af733e5dde61c Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 19 Dec 2025 16:28:31 +1100 Subject: [PATCH 29/89] Extract PCGR errors from log files --- bolt/util.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/bolt/util.py b/bolt/util.py index 726ffc9..7d9d890 100644 --- a/bolt/util.py +++ b/bolt/util.py @@ -31,6 +31,10 @@ def execute_command(command, log_file_path=None): # Open the log file if provided log_file = log_file_path.open('a', encoding='utf-8') if log_file_path else None + # Set environment to suppress bash libtinfo.so.6 warnings + env = subprocess.os.environ.copy() + env['BASH_SILENCE_DEPRECATION_WARNING'] = '1' + # Launch process with combined stdout and stderr streams, and line buffering enabled. process = subprocess.Popen( prepared_command, @@ -40,7 +44,8 @@ def execute_command(command, log_file_path=None): stderr=subprocess.STDOUT, text=True, encoding='utf-8', - bufsize=1 # line buffered + bufsize=1, # line buffered + env=env ) output_lines = [] @@ -48,11 +53,13 @@ def execute_command(command, log_file_path=None): with process.stdout: for line in iter(process.stdout.readline, ''): if line: - logger.info(line.strip()) - output_lines.append(line) - if log_file: - log_file.write(line) - log_file.flush() # flush immediately for real-time logging + # Filter out bash libtinfo.so.6 warnings + if 'libtinfo.so.6: no version information available' not in line: + logger.info(line.strip()) + output_lines.append(line) + if log_file: + log_file.write(line) + log_file.flush() # flush immediately for real-time logging process.wait() # wait for the process to complete if log_file: From 9bf22f5573451e420b16871af7cf1cb17e734dbc Mon Sep 17 00:00:00 2001 From: Quentin Clayssen <37511834+qclayssen@users.noreply.github.com> Date: Wed, 21 Jan 2026 14:32:52 +1100 Subject: [PATCH 30/89] Update bolt/workflows/other/cancer_report.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- bolt/workflows/other/cancer_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bolt/workflows/other/cancer_report.py b/bolt/workflows/other/cancer_report.py index 7e2d70f..7717afd 100644 --- a/bolt/workflows/other/cancer_report.py +++ b/bolt/workflows/other/cancer_report.py @@ -116,7 +116,7 @@ def entry(ctx, **kwargs): \ --img_dir {output_image_dir}/ \ --result_outdir {output_table_dir}/ \ - --out_file {output_dir}/{kwargs['tumor_name']}.cancer_report.html + --out_file {output_dir}/{kwargs["tumor_name"]}.cancer_report.html ''' util.execute_command(command) From 599849da5a9b4b69b7fe6c8285c2deb1816a4a88 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen <37511834+qclayssen@users.noreply.github.com> Date: Wed, 21 Jan 2026 14:33:17 +1100 Subject: [PATCH 31/89] Update bolt/common/pcgr.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- bolt/common/pcgr.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bolt/common/pcgr.py b/bolt/common/pcgr.py index ecbe6b1..feafc5d 100644 --- a/bolt/common/pcgr.py +++ b/bolt/common/pcgr.py @@ -8,7 +8,6 @@ import shutil import tempfile import logging -import concurrent.futures import cyvcf2 From 09a3a247ad2f6a9762321f7fd7e28abd0f7939c6 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen <37511834+qclayssen@users.noreply.github.com> Date: Thu, 19 Feb 2026 15:47:48 +1100 Subject: [PATCH 32/89] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .bumpversion.cfg | 2 +- bolt/common/constants.py | 35 ++++++++++++++----------- bolt/common/pcgr.py | 4 +-- bolt/util.py | 5 ++-- bolt/workflows/smlv_somatic/annotate.py | 2 +- bolt/workflows/smlv_somatic/filter.py | 2 +- bolt/workflows/smlv_somatic/rescue.py | 3 +-- pyproject.toml | 2 +- 8 files changed, 28 insertions(+), 27 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index fe18769..4bae614 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.2.17 +current_version = 0.3.0 commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P[a-z0-9+]+) diff --git a/bolt/common/constants.py b/bolt/common/constants.py index a7a207d..1e2d864 100644 --- a/bolt/common/constants.py +++ b/bolt/common/constants.py @@ -227,7 +227,10 @@ def namespace(self): }, VcfFilter.PCGR_COUNT_LIMIT: { - 'Description': 'Manually filtered to meet PCGR 500,000 variant limit', + 'Description': ( + f'Manually filtered to {MAX_SOMATIC_VARIANTS} variants to stay below the ' + 'PCGR 500,000 variant limit' + ), }, # INFO @@ -279,21 +282,21 @@ def namespace(self): }, VcfInfo.PCGR_CSQ: { 'Number': '.', - 'Type': 'String', - 'Description': ( - 'Consequence annotations from Ensembl VEP. Format: ' - 'Allele|Consequence|IMPACT|SYMBOL|Gene|Feature_type|Feature|BIOTYPE|EXON|INTRON|HGVSc|' - 'HGVSp|cDNA_position|CDS_position|Protein_position|Amino_acids|Codons|Existing_variation|' - 'ALLELE_NUM|DISTANCE|STRAND|FLAGS|PICK|VARIANT_CLASS|SYMBOL_SOURCE|HGNC_ID|CANONICAL|' - 'MANE|MANE_SELECT|MANE_PLUS_CLINICAL|TSL|APPRIS|CCDS|ENSP|SWISSPROT|TREMBL|UNIPARC|' - 'UNIPROT_ISOFORM|RefSeq|DOMAINS|HGVS_OFFSET|gnomADe_AF|gnomADe_AFR_AF|gnomADe_AMR_AF|' - 'gnomADe_ASJ_AF|gnomADe_EAS_AF|gnomADe_FIN_AF|gnomADe_MID_AF|gnomADe_NFE_AF|' - 'gnomADe_REMAINING_AF|gnomADe_SAS_AF|gnomADg_AF|gnomADg_AFR_AF|gnomADg_AMI_AF|' - 'gnomADg_AMR_AF|gnomADg_ASJ_AF|gnomADg_EAS_AF|gnomADg_FIN_AF|gnomADg_MID_AF|' - 'gnomADg_NFE_AF|gnomADg_REMAINING_AF|gnomADg_SAS_AF|CLIN_SIG|SOMATIC|PHENO|CHECK_REF|' - 'MOTIF_NAME|MOTIF_POS|HIGH_INF_POS|MOTIF_SCORE_CHANGE|TRANSCRIPTION_FACTORS|NearestExonJB|' - 'MaxEntScan_alt|MaxEntScan_diff|MaxEntScan_ref' - ), + 'Type': 'String', + 'Description': ( + 'Consequence annotations from Ensembl VEP. Format: ' + 'Allele|Consequence|IMPACT|SYMBOL|Gene|Feature_type|Feature|BIOTYPE|EXON|INTRON|HGVSc|' + 'HGVSp|cDNA_position|CDS_position|Protein_position|Amino_acids|Codons|Existing_variation|' + 'ALLELE_NUM|DISTANCE|STRAND|FLAGS|PICK|VARIANT_CLASS|SYMBOL_SOURCE|HGNC_ID|CANONICAL|' + 'MANE|MANE_SELECT|MANE_PLUS_CLINICAL|TSL|APPRIS|CCDS|ENSP|SWISSPROT|TREMBL|UNIPARC|' + 'UNIPROT_ISOFORM|RefSeq|DOMAINS|HGVS_OFFSET|gnomADe_AF|gnomADe_AFR_AF|gnomADe_AMR_AF|' + 'gnomADe_ASJ_AF|gnomADe_EAS_AF|gnomADe_FIN_AF|gnomADe_MID_AF|gnomADe_NFE_AF|' + 'gnomADe_REMAINING_AF|gnomADe_SAS_AF|gnomADg_AF|gnomADg_AFR_AF|gnomADg_AMI_AF|' + 'gnomADg_AMR_AF|gnomADg_ASJ_AF|gnomADg_EAS_AF|gnomADg_FIN_AF|gnomADg_MID_AF|' + 'gnomADg_NFE_AF|gnomADg_REMAINING_AF|gnomADg_SAS_AF|CLIN_SIG|SOMATIC|PHENO|CHECK_REF|' + 'MOTIF_NAME|MOTIF_POS|HIGH_INF_POS|MOTIF_SCORE_CHANGE|TRANSCRIPTION_FACTORS|NearestExonJB|' + 'MaxEntScan_alt|MaxEntScan_diff|MaxEntScan_ref' + ), }, VcfInfo.PCGR_MUTATION_HOTSPOT: { 'Number': '.', diff --git a/bolt/common/pcgr.py b/bolt/common/pcgr.py index feafc5d..bf4d0f7 100644 --- a/bolt/common/pcgr.py +++ b/bolt/common/pcgr.py @@ -1,4 +1,4 @@ -import collections + import csv import functools import gzip @@ -553,7 +553,7 @@ def split_vcf(input_vcf, output_dir, *, max_variants=None): logger.info(f"VCF file split into {len(chunk_files)} chunks.") return chunk_files -def run_somatic_chunck(vcf_chunks, pcgr_data_dir, vep_dir, output_dir, pcgr_output_dir, max_threads, pcgr_conda, pcgrr_conda): +def run_somatic_chunk(vcf_chunks, pcgr_data_dir, vep_dir, output_dir, pcgr_output_dir, max_threads, pcgr_conda, pcgrr_conda): pcgr_tsv_files = [] pcgr_vcf_files = [] diff --git a/bolt/util.py b/bolt/util.py index 7d9d890..7d065cd 100644 --- a/bolt/util.py +++ b/bolt/util.py @@ -31,9 +31,8 @@ def execute_command(command, log_file_path=None): # Open the log file if provided log_file = log_file_path.open('a', encoding='utf-8') if log_file_path else None - # Set environment to suppress bash libtinfo.so.6 warnings + # Prepare environment for subprocess (inherit current environment) env = subprocess.os.environ.copy() - env['BASH_SILENCE_DEPRECATION_WARNING'] = '1' # Launch process with combined stdout and stderr streams, and line buffering enabled. process = subprocess.Popen( @@ -276,7 +275,7 @@ def check_annotation_headers(info_field_map, vcf_fp): # Remove leading and trailing quotes from source header_src_description_unquoted = header_src_entry['Description'].strip('"') try: - assert header_src_description_unquoted == header_dst_entry['Description'] + assert header_src_description_unquoted == header_dst_entry['Description'] except AssertionError: print(f'Header description mismatch for {header_dst.value}') print(f' src: {header_src_description_unquoted}') diff --git a/bolt/workflows/smlv_somatic/annotate.py b/bolt/workflows/smlv_somatic/annotate.py index 32b85b3..01d28b2 100644 --- a/bolt/workflows/smlv_somatic/annotate.py +++ b/bolt/workflows/smlv_somatic/annotate.py @@ -107,7 +107,7 @@ def entry(ctx, **kwargs): if total_variants > chunk_size: vcf_chunks = pcgr.split_vcf(pcgr_prep_fp, output_dir, max_variants=chunk_size) - pcgr_tsv_fp, pcgr_vcf_fp = pcgr.run_somatic_chunck( + pcgr_tsv_fp, pcgr_vcf_fp = pcgr.run_somatic_chunk( vcf_chunks, kwargs['pcgr_data_dir'], kwargs['vep_dir'], diff --git a/bolt/workflows/smlv_somatic/filter.py b/bolt/workflows/smlv_somatic/filter.py index c820385..7fff6fc 100644 --- a/bolt/workflows/smlv_somatic/filter.py +++ b/bolt/workflows/smlv_somatic/filter.py @@ -265,7 +265,7 @@ def get_record_value(record, key, *, default=None): value = record.INFO.get(key) if value is None: return default - if value=='' or value=='.': + if value == '' or value == '.': return default if not isinstance(value, (int, float, str)): logger.error(f'record ID: {record.ID} INFO value for {key}: {value} ({type(value)}, not int, float, str)') diff --git a/bolt/workflows/smlv_somatic/rescue.py b/bolt/workflows/smlv_somatic/rescue.py index 4a9973b..ff69e6e 100644 --- a/bolt/workflows/smlv_somatic/rescue.py +++ b/bolt/workflows/smlv_somatic/rescue.py @@ -113,8 +113,7 @@ def annotate_existing_sage_calls(input_fp, tumor_name, sage_vcf_fp, output_dir): # Get input file handle input_fh = cyvcf2.VCF(input_fp) - # Get input file handle - # Perform consistency check on the VCF header + # Define expected SAGE annotations used for VCF header consistency check info_field_map_sage = { constants.VcfInfo.SAGE_HOTSPOT: 'SAGE_HOTSPOT', constants.VcfInfo.SAGE_NOVEL: 'SAGE_NOVEL', diff --git a/pyproject.toml b/pyproject.toml index e7325a3..b9c2128 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ include = ["bolt*"] [project] name = "bolt" -version = "0.2.17" +version = "0.3.0" authors = [ {name = "Stephen Watts", email = "stephen.watts@umccr.org"}, ] From dbd500f5df52611edc33e47fb6e8631b8da35d83 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Thu, 19 Feb 2026 16:06:06 +1100 Subject: [PATCH 33/89] fix typo --- bolt/workflows/other/cancer_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bolt/workflows/other/cancer_report.py b/bolt/workflows/other/cancer_report.py index 7717afd..7e2d70f 100644 --- a/bolt/workflows/other/cancer_report.py +++ b/bolt/workflows/other/cancer_report.py @@ -116,7 +116,7 @@ def entry(ctx, **kwargs): \ --img_dir {output_image_dir}/ \ --result_outdir {output_table_dir}/ \ - --out_file {output_dir}/{kwargs["tumor_name"]}.cancer_report.html + --out_file {output_dir}/{kwargs['tumor_name']}.cancer_report.html ''' util.execute_command(command) From 3d2d7b1deadff7b36b3c9af3939adce44cb5bc77 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Thu, 19 Feb 2026 16:06:22 +1100 Subject: [PATCH 34/89] remove duplicate fonction --- bolt/util.py | 49 ------------------------------------------------- 1 file changed, 49 deletions(-) diff --git a/bolt/util.py b/bolt/util.py index 7d065cd..bd89c13 100644 --- a/bolt/util.py +++ b/bolt/util.py @@ -121,55 +121,6 @@ def get_qualified_vcf_annotation(anno_enum): assert anno_enum in constants.VcfInfo or anno_enum in constants.VcfFormat return f'{anno_enum.namespace}/{anno_enum.value}' -def split_vcf(input_vcf, output_dir): - """ - Splits a VCF file into multiple chunks, each containing up to max_variants variants. - Each chunk includes the VCF header. - Ensures no overlapping positions between chunks. - """ - output_dir = pathlib.Path(output_dir / "vcf_chunks") - output_dir.mkdir(parents=True, exist_ok=True) - - chunk_files = [] - chunk_number = 1 - variant_count = 0 - input_vcf = pathlib.Path(input_vcf) - base_filename = input_vcf.stem - chunk_filename = output_dir / f"{base_filename}_chunk{chunk_number}.vcf" - chunk_files.append(chunk_filename) - - # Open the input VCF using cyvcf2 - vcf_in = cyvcf2.VCF(input_vcf) - # Create a new VCF file for the first chunk - vcf_out = cyvcf2.Writer(str(chunk_filename), vcf_in) - - last_position = None - - for record in vcf_in: - current_position = record.POS - # Check if we need to start a new chunk - if variant_count >= constants.MAX_SOMATIC_VARIANTS and (last_position is None or current_position != last_position): - # Close the current chunk file and start a new one - vcf_out.close() - chunk_number += 1 - chunk_filename = output_dir / f"{base_filename}_chunk{chunk_number}.vcf" - chunk_files.append(chunk_filename) - vcf_out = cyvcf2.Writer(str(chunk_filename), vcf_in) - variant_count = 0 - - # Write the record to the current chunk - vcf_out.write_record(record) - variant_count += 1 - last_position = current_position - - # Close the last chunk file - vcf_out.close() - vcf_in.close() - - logger.info(f"VCF file split into {len(chunk_files)} chunks.") - - return chunk_files - def merge_tsv_files(tsv_files, merged_tsv_fp): """ Merge gzipped TSV files into a single gzipped TSV. From 15f024718c26a14d74d4d27b991668930353aa51 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 20 Feb 2026 11:32:33 +1100 Subject: [PATCH 35/89] fix typo --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25ce706..f026d5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,6 @@ - [3](https://github.com/scwatts/bolt/pull/3) - Improve PCGR / CPSR argument handling -- [6](https://github.com/umccr/bolt/pull/6) - Change oncoanalyser v2.0.0 uptade, with switch sv caller from GRIPSS to eSVee +- [6](https://github.com/umccr/bolt/pull/6) - Change oncoanalyser v2.0.0 update, with switch sv caller from GRIPSS to eSVee - [9](https://github.com/umccr/bolt/pull/9) Add hypermutation sample handling \ No newline at end of file From 8fa85ef5de06fd8be287814b00206cccc7355fc7 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 20 Feb 2026 14:07:47 +1100 Subject: [PATCH 36/89] add check pcgr_hotspot_present value --- bolt/workflows/smlv_somatic/filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bolt/workflows/smlv_somatic/filter.py b/bolt/workflows/smlv_somatic/filter.py index 7fff6fc..aa4f7c4 100644 --- a/bolt/workflows/smlv_somatic/filter.py +++ b/bolt/workflows/smlv_somatic/filter.py @@ -214,7 +214,7 @@ def set_filter_data(record, tumor_index): if ( hmf_present is not None or - pcgr_hotspot_present or + pcgr_hotspot_present is not None or any(e in clinvar_clinsigs for e in constants.CLINVAR_CLINSIGS_RESCUE) or tcga_pancancer_count >= constants.MIN_TCGA_PANCANCER_COUNT_RESCUE ): From 31dababa41aae4e23a761e5dcc5cf6dd8354fc26 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Thu, 7 May 2026 16:17:40 +1000 Subject: [PATCH 37/89] add is_hypermutated flag to variant counts process JSON --- bolt/workflows/smlv_somatic/report.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bolt/workflows/smlv_somatic/report.py b/bolt/workflows/smlv_somatic/report.py index 0954cc3..2ef50ca 100644 --- a/bolt/workflows/smlv_somatic/report.py +++ b/bolt/workflows/smlv_somatic/report.py @@ -301,6 +301,7 @@ def count_variant_process(vcf_fp): if not record.FILTER or rescued_filters: counts['filter_pass'] += 1 + counts['is_hypermutated'] = counts['filter_pass'] > constants.MAX_SOMATIC_VARIANTS return counts From f3d557c61ddbb60940901454fbfaf5637d9dce25 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Tue, 12 May 2026 10:54:26 +1000 Subject: [PATCH 38/89] update PCGR_TIERS_FILTERING --- bolt/common/constants.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/bolt/common/constants.py b/bolt/common/constants.py index 1e2d864..0dba0b9 100644 --- a/bolt/common/constants.py +++ b/bolt/common/constants.py @@ -44,12 +44,14 @@ ################################ ## Hypermutated report filter ## ################################ +# Values match short forms written by transfer_annotations_somatic() ('1'=TIER_1, ..., 'N'=NONCODING). +# Order is lowest clinical priority first: NONCODING dropped before TIER_1. PCGR_TIERS_FILTERING = ( - 'TIER_1', - 'TIER_2', - 'TIER_3', - 'TIER_4', - 'NONCODING', + 'N', + '4', + '3', + '2', + '1', ) VEP_IMPACTS_FILTER = ( From 184e170d0dccd4fe2577fd469c58533b4bcd1db4 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Tue, 12 May 2026 14:57:17 +1000 Subject: [PATCH 39/89] Fix PCGR hypermutated filtering, add exhaustive tests Corrects the tier ordering logic and filtering in hypermutated sample handling to ensure NONCODING variants are dropped before TIER_1, preventing excessive variant counts. Refines error handling, improves logging and VCF writing, and clarifies filter traceability. Adds comprehensive unit and integration tests to verify all edge cases in tiered variant selection and CSQ parsing. Relates to improved reliability for high-mutation samples. --- bolt/common/pcgr.py | 25 +- bolt/workflows/smlv_somatic/report.py | 57 +++-- tests/test_pcgr_hypermutated.py | 327 ++++++++++++++++++++++++++ 3 files changed, 384 insertions(+), 25 deletions(-) create mode 100644 tests/test_pcgr_hypermutated.py diff --git a/bolt/common/pcgr.py b/bolt/common/pcgr.py index bf4d0f7..a7bc9e2 100644 --- a/bolt/common/pcgr.py +++ b/bolt/common/pcgr.py @@ -116,8 +116,13 @@ def get_minimal_header(input_fh): return '\n'.join([filetype_line, *chrom_lines, *format_lines, column_line]) -def run_somatic(input_fp, pcgr_refdata_dir, vep_dir, output_dir, chunk_nbr=None, threads=1, pcgr_conda=None, pcgrr_conda=None, purity=None, ploidy=None, sample_id=None): +def run_somatic(input_fp, pcgr_refdata_dir, vep_dir, output_dir, chunk_nbr=None, threads=1, pcgr_threads=4, pcgr_conda=None, pcgrr_conda=None, purity=None, ploidy=None, sample_id=None, disable_estimates=False): + # threads: Nextflow process-level resource allocation (not wired to PCGR internals) + # pcgr_threads: PCGR-internal concurrency (vcfanno workers + VEP forks) + pcgr_threads = max(1, int(pcgr_threads)) + vcfanno_threads = pcgr_threads + vep_forks = min(8, max(2, pcgr_threads)) output_dir = output_dir / f"pcgr_{chunk_nbr}" if chunk_nbr is not None else output_dir @@ -142,11 +147,9 @@ def run_somatic(input_fp, pcgr_refdata_dir, vep_dir, output_dir, chunk_nbr=None, f'--control_af_tag NORMAL_AF', f'--genome_assembly grch38', f'--assay WGS', - f'--estimate_signatures', - f'--estimate_msi', - f'--estimate_tmb', - f'--vcfanno_n_proc 4', - f'--vep_n_forks 4', + *([] if disable_estimates else ['--estimate_signatures', '--estimate_msi', '--estimate_tmb']), + f'--vcfanno_n_proc {vcfanno_threads}', + f'--vep_n_forks {vep_forks}', f'--vep_pick_order biotype,rank,appris,tsl,ccds,canonical,length,mane_plus_clinical,mane_select', ] @@ -211,7 +214,9 @@ def run_somatic(input_fp, pcgr_refdata_dir, vep_dir, output_dir, chunk_nbr=None, return pcgr_tsv_fp, pcgr_vcf_fp -def run_germline(input_fp, panel_fp, pcgr_refdata_dir, vep_dir, output_dir, threads=1, pcgr_conda=None, pcgrr_conda=None, sample_id=None): +def run_germline(input_fp, panel_fp, pcgr_refdata_dir, vep_dir, output_dir, threads=1, pcgr_threads=4, pcgr_conda=None, pcgrr_conda=None, sample_id=None): + # threads: Nextflow process-level resource allocation (not wired to CPSR internals) + # pcgr_threads: CPSR-internal concurrency (vcfanno workers) if not sample_id: sample_id = 'nosampleset' @@ -238,7 +243,7 @@ def run_germline(input_fp, panel_fp, pcgr_refdata_dir, vep_dir, output_dir, thre f'--custom_list_name umccr_germline_panel', f'--pop_gnomad global', f'--classify_all', - f'--vcfanno_n_proc {threads}', + f'--vcfanno_n_proc {pcgr_threads}', f'--vep_pick_order biotype,rank,appris,tsl,ccds,canonical,length,mane_plus_clinical,mane_select', ] @@ -568,13 +573,13 @@ def run_somatic_chunk(vcf_chunks, pcgr_data_dir, vep_dir, output_dir, pcgr_outpu merged_vcf_fp, merged_tsv_fp = merging_pcgr_files(output_dir, pcgr_vcf_files, pcgr_tsv_files) return merged_tsv_fp, merged_vcf_fp -def merging_pcgr_files(output_dir, pcgr_vcf_files, pcgr_tsv_fp): +def merging_pcgr_files(output_dir, pcgr_vcf_files, pcgr_tsv_files): pcgr_dir = pathlib.Path(output_dir) / 'pcgr' pcgr_dir.mkdir(exist_ok=True) # Merge all TSV files into a single file in the pcgr directory merged_tsv_fp = pcgr_dir / "nosampleset.pcgr_acmg.grch38.snvs_indels.tiers.tsv.gz" - util.merge_tsv_files(pcgr_tsv_fp, merged_tsv_fp) + util.merge_tsv_files(pcgr_tsv_files, merged_tsv_fp) # Step 5: Merge all VCF files into a single file in the pcgr directory merged_vcf_path = pcgr_dir / "nosampleset.pcgr.grch38.pass" diff --git a/bolt/workflows/smlv_somatic/report.py b/bolt/workflows/smlv_somatic/report.py index 2ef50ca..a65efea 100644 --- a/bolt/workflows/smlv_somatic/report.py +++ b/bolt/workflows/smlv_somatic/report.py @@ -70,7 +70,7 @@ def entry(ctx, **kwargs): # Variant type counts # NOTE(SW): this is intended to preserve counts in the MultiQC report variant_counts_types_dragen = count_variant_types(kwargs['vcf_dragen_fp']) - variant_counts_types_bolt= count_variant_types(kwargs['vcf_fp']) + variant_counts_types_bolt = count_variant_types(kwargs['vcf_fp']) # NOTE(SW): using pass variants only for now @@ -113,6 +113,8 @@ def entry(ctx, **kwargs): fh.write('\n') # PCGR report + purple_data = parse_purple_purity_file(kwargs['purple_purity_fp']) + if variant_counts_process['filter_pass'] <= constants.MAX_SOMATIC_VARIANTS: pcgr_input_vcf_fp = kwargs['vcf_fp'] else: @@ -123,8 +125,6 @@ def entry(ctx, **kwargs): output_dir, ) - purple_data = parse_purple_purity_file(kwargs['purple_purity_fp']) - pcgr_prep_fp = pcgr.prepare_vcf_somatic( pcgr_input_vcf_fp, kwargs['tumor_name'], @@ -163,7 +163,7 @@ def bcftools_stats_prepare(input_fp, tumor_name, output_dir): elif record.INFO.get('SAGE_NOVEL') is not None: record.QUAL = None else: - assert False + raise AssertionError(f'Record at {record.CHROM}:{record.POS} has neither SQ nor SAGE_NOVEL — cannot determine QUAL') output_fh.write_record(record) @@ -173,8 +173,8 @@ def bcftools_stats_prepare(input_fp, tumor_name, output_dir): def run_bcftools_stats(input_fp, tumor_name, output_dir): output_fp = output_dir / f'{tumor_name}.somatic.bcftools_stats.txt' command = fr''' - bcftools stats {input_fp} | \ - sed '6 s#{input_fp}$#{tumor_name}#' > {output_fp} + bcftools stats '{input_fp}' | \ + sed '6 s#{input_fp}$#{tumor_name}#' > '{output_fp}' ''' util.execute_command(command) @@ -306,6 +306,18 @@ def count_variant_process(vcf_fp): def select_pcgr_variants(vcf_fp, cancer_genes_fp, tumor_name, output_dir): + """Filter variants for hypermutated samples to stay below PCGR's 500k limit. + + Retained variants (hotspot / panel) are skipped for tiered filtering. The remaining + variants are classified by (tier, impact, region) and categories are dropped in + priority order — NONCODING first, TIER_1 last — until the count falls within + MAX_SOMATIC_VARIANTS. Raises RuntimeError if the limit cannot be reached (i.e. + retained variants alone exceed it). + + Returns the path to the pass VCF (variants that survived filtering). A second + traceability VCF with FILTER=PCGR_count_limit set on dropped variants is written + alongside it. + """ # Annotate variants in UMCCR somatic gene panel fp_annotated_out = output_dir / f'{tumor_name}.umccr_panel_variants_annotated.vcf.gz' util.execute_command(fr''' @@ -319,44 +331,59 @@ def select_pcgr_variants(vcf_fp, cancer_genes_fp, tumor_name, output_dir): # Set filter category for each variant variants_sorted = collections.defaultdict(list) + variant_count = 0 + for variant_count, variant in enumerate(cyvcf2.VCF(fp_annotated_out), 1): + variant_repr = pcgr.get_variant_repr(variant) + if any(variant.INFO.get(e) for e in constants.RETAIN_FIELDS_FILTERING): continue data = pcgr.get_variant_filter_data(variant) variant_filter = pcgr.determine_filter(data) - assert variant_filter + if not variant_filter: + raise AssertionError( + f'determine_filter returned no category for variant {variant_repr} (data={data})' + ) filter_category = (data['tier'], *variant_filter) - variant_repr = pcgr.get_variant_repr(variant) variants_sorted[filter_category].append(variant_repr) - # Determine the set of filter categories to come under the PCGR 500,000 variant threshold filter_sum = 0 filter_categories = list() for key in pcgr.get_ordering(): - if (variant_count - filter_sum) <= constants.MAX_SOMATIC_VARIANTS: break - filter_sum += len(variants_sorted.get(key, [])) filter_categories.append(key) - # Set FILTERS and write out records filter_variants = set() for key in filter_categories: filter_variants.update(variants_sorted[key]) + expected_output = variant_count - len(filter_variants) + if expected_output > constants.MAX_SOMATIC_VARIANTS: + raise RuntimeError( + f'select_pcgr_variants failed to cap variants for {tumor_name}: ' + f'{expected_output} > {constants.MAX_SOMATIC_VARIANTS}' + ) + + logger.info('%s: select_pcgr_variants total=%d filtered=%d output=%d', + tumor_name, variant_count, len(filter_variants), expected_output) + + # Write passing variants; write all variants (with FILTER set) for traceability. + # Re-opening the original vcf_fp (not fp_annotated_out) so that the temporary + # PANEL annotation used for tiered filtering is not propagated to PCGR input. fh_in = cyvcf2.VCF(vcf_fp) util.add_vcf_header_entry(fh_in, constants.VcfFilter.PCGR_COUNT_LIMIT) - # NOTE(SW): creating an additional VCF with all records for traceability + # Additional VCF with all records for traceability fp_out = output_dir / f'{tumor_name}.pcgr_hypermutated.pass.vcf.gz' fp_set_out = output_dir / f'{tumor_name}.pcgr_hypermutated.filters_set.vcf.gz' - fh_out = cyvcf2.Writer(fp_out, fh_in) - fh_set_out = cyvcf2.Writer(fp_set_out, fh_in) + fh_out = cyvcf2.Writer(fp_out, fh_in, 'wz') + fh_set_out = cyvcf2.Writer(fp_set_out, fh_in, 'wz') for variant in fh_in: variant_repr = pcgr.get_variant_repr(variant) diff --git a/tests/test_pcgr_hypermutated.py b/tests/test_pcgr_hypermutated.py new file mode 100644 index 0000000..3dd786a --- /dev/null +++ b/tests/test_pcgr_hypermutated.py @@ -0,0 +1,327 @@ +"""Tests for hypermutated sample handling — tier ordering fix and variant trimming.""" +import pathlib +import shutil +import tempfile +import unittest +from unittest.mock import patch + +import cyvcf2 + +import bolt.common.constants as constants +import bolt.common.pcgr as pcgr +import bolt.util as util +import bolt.workflows.smlv_somatic.report as report_mod + + +# Minimal CSQ: only tokens[1] (consequence) is read by get_impacts() +def _csq(consequence): + return f'A|{consequence}|.|.|.|.|.|.|.|.|.|.|.|.|.|.|.|.|.|.|.' + + +# Minimal VCF header with all INFO fields used by select_pcgr_variants +HEADER = ( + '##fileformat=VCFv4.2\n' + '##FILTER=\n' + '##INFO=\n' + '##INFO=\n' + '##INFO=\n' + '##INFO=\n' + '##INFO=\n' + '##INFO=\n' + '##INFO=\n' + '##INFO=\n' + '##contig=\n' + '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n' +) + + +def _write_vcf(path, variants): + with open(path, 'w') as fh: + fh.write(HEADER) + for pos, info in variants: + fh.write(f'chr1\t{pos}\t.\tA\tT\t.\tPASS\t{info}\n') + + +def _count_vcf(fp): + return sum(1 for _ in cyvcf2.VCF(str(fp))) + + +def _make_variant(info_str): + """Return a cyvcf2 Variant built from info_str using the test VCF header.""" + with tempfile.TemporaryDirectory() as tmp: + vcf_path = pathlib.Path(tmp) / 'test.vcf' + _write_vcf(vcf_path, [(100, info_str)]) + return list(cyvcf2.VCF(str(vcf_path)))[0] + + +class TestTierOrdering(unittest.TestCase): + """Verify the PCGR_TIERS_FILTERING fix: values and priority order.""" + + def test_noncoding_filtered_before_tier1(self): + """N (NONCODING) entries must all precede '1' (TIER_1) entries in get_ordering().""" + ordering = pcgr.get_ordering() + tiers = [key[0] for key in ordering] + n_idx = [i for i, t in enumerate(tiers) if t == 'N'] + t1_idx = [i for i, t in enumerate(tiers) if t == '1'] + self.assertTrue(n_idx, 'No NONCODING (N) entries in get_ordering()') + self.assertTrue(t1_idx, 'No TIER_1 (1) entries in get_ordering()') + self.assertLess(max(n_idx), min(t1_idx), + 'All NONCODING entries must precede all TIER_1 entries') + + def test_no_long_form_tier_values(self): + """PCGR_TIERS_FILTERING must use short forms ('1'-'4', 'N'), not 'TIER_1' etc.""" + for v in constants.PCGR_TIERS_FILTERING: + self.assertNotIn('TIER_', v, + f"Found long-form tier value '{v}' — must be short form") + + def test_priority_order(self): + """Full ordering: N before 4 before 3 before 2 before 1.""" + expected = ('N', '4', '3', '2', '1') + self.assertEqual(constants.PCGR_TIERS_FILTERING, expected) + + +class TestSelectPcgrVariants(unittest.TestCase): + """Integration tests for select_pcgr_variants() trimming logic.""" + + def _run(self, variants, limit, tmp): + """Run select_pcgr_variants with a small MAX_SOMATIC_VARIANTS limit.""" + vcf_fp = pathlib.Path(tmp) / 'input.vcf' + _write_vcf(vcf_fp, variants) + cancer_genes = pathlib.Path(tmp) / 'genes.bed' + cancer_genes.write_text('chr1\t1\t9999999\n') + + # Mock bcftools annotate: copy input to the expected output path + orig_execute = util.execute_command + def fake_execute(cmd, **_): + import re + m = re.search(r'--output\s+(\S+)', cmd) + if m and 'bcftools annotate' in cmd: + shutil.copy(str(vcf_fp), m.group(1)) + else: + orig_execute(cmd) + + with patch('bolt.common.constants.MAX_SOMATIC_VARIANTS', limit), \ + patch('bolt.util.execute_command', side_effect=fake_execute): + out_fp = report_mod.select_pcgr_variants( + vcf_fp, cancer_genes, 'TUMOR', pathlib.Path(tmp) + ) + return _count_vcf(out_fp) + + def test_output_within_limit(self): + """Output must never exceed MAX_SOMATIC_VARIANTS.""" + with tempfile.TemporaryDirectory() as tmp: + # 15 variants: 2 hotspot + 5 TIER_1 + 5 TIER_3 + 3 NONCODING + v = [] + for i in range(1, 3): # hotspot + v.append((i*10, f'HMF_HOTSPOT;PCGR_ACTIONABILITY_TIER=1;PCGR_CSQ={_csq("intron_variant")}')) + for i in range(3, 8): # TIER_1 intronic + v.append((i*10, f'PCGR_ACTIONABILITY_TIER=1;PCGR_CSQ={_csq("intron_variant")}')) + for i in range(8, 13): # TIER_3 intronic + v.append((i*10, f'PCGR_ACTIONABILITY_TIER=3;PCGR_CSQ={_csq("intron_variant")}')) + for i in range(13, 16): # NONCODING intergenic + v.append((i*10, f'PCGR_ACTIONABILITY_TIER=N;PCGR_CSQ={_csq("intergenic_variant")}')) + count = self._run(v, limit=10, tmp=tmp) + self.assertLessEqual(count, 10) + + def test_noncoding_dropped_before_tier1(self): + """With limit = total - 3, the 3 NONCODING variants should be dropped (not TIER_1).""" + with tempfile.TemporaryDirectory() as tmp: + v = [] + for i in range(1, 6): # 5 TIER_1 intronic + v.append((i*10, f'PCGR_ACTIONABILITY_TIER=1;PCGR_CSQ={_csq("intron_variant")}')) + for i in range(6, 9): # 3 NONCODING intergenic + v.append((i*10, f'PCGR_ACTIONABILITY_TIER=N;PCGR_CSQ={_csq("intergenic_variant")}')) + # limit=5: should drop the 3 NONCODING to get to 5 + count = self._run(v, limit=5, tmp=tmp) + self.assertEqual(count, 5) + + def test_hotspots_never_dropped_by_tiered_filter(self): + """Hotspot variants must survive tiered filtering. + + Note: HMF_HOTSPOT is not in RETAIN_FIELDS_FILTERING — these variants survive because + they are TIER_1 (highest priority), not via the hotspot retention path. + """ + with tempfile.TemporaryDirectory() as tmp: + v = [] + for i in range(1, 3): # 2 HMF_HOTSPOT TIER_1 variants (survive via tier priority) + v.append((i*10, f'HMF_HOTSPOT;PCGR_ACTIONABILITY_TIER=1;PCGR_CSQ={_csq("intron_variant")}')) + for i in range(3, 13): # 10 NONCODING (should all be filtered) + v.append((i*10, f'PCGR_ACTIONABILITY_TIER=N;PCGR_CSQ={_csq("intergenic_variant")}')) + # limit=2: only the 2 TIER_1 variants should remain + count = self._run(v, limit=2, tmp=tmp) + self.assertEqual(count, 2) + + def test_all_within_limit_nothing_filtered(self): + """When total variants are below the limit, nothing is dropped.""" + with tempfile.TemporaryDirectory() as tmp: + v = [(i*10, f'PCGR_ACTIONABILITY_TIER=N;PCGR_CSQ={_csq("intergenic_variant")}') + for i in range(1, 6)] # 5 NONCODING + count = self._run(v, limit=10, tmp=tmp) + self.assertEqual(count, 5) + + def test_retained_variants_bypass_tiered_filter(self): + """Variants with PANEL or SAGE_HOTSPOT bypass tiered filtering and always survive.""" + with tempfile.TemporaryDirectory() as tmp: + v = [] + for i in range(1, 3): # 2 SAGE_HOTSPOT + v.append((i*10, 'SAGE_HOTSPOT')) + for i in range(3, 5): # 2 PANEL-only + v.append((i*10, 'PANEL')) + for i in range(5, 15): # 10 NONCODING that get dropped + v.append((i*10, f'PCGR_ACTIONABILITY_TIER=N;PCGR_CSQ={_csq("intergenic_variant")}')) + count = self._run(v, limit=4, tmp=tmp) + self.assertEqual(count, 4) + + def test_filters_set_vcf_marks_dropped_variants(self): + """The traceability VCF marks filtered-out variants with PCGR_count_limit. + + The function drops entire categories, so we need two distinct categories: + - 3 TIER_1 intronic (high priority — kept) + - 2 NONCODING intergenic (lowest priority — dropped as a whole category) + """ + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + vcf_fp = tmp_path / 'input.vcf' + v = [] + for i in range(1, 4): # 3 TIER_1 intronic — kept + v.append((i*10, f'PCGR_ACTIONABILITY_TIER=1;PCGR_CSQ={_csq("intron_variant")}')) + for i in range(4, 6): # 2 NONCODING intergenic — dropped whole category + v.append((i*10, f'PCGR_ACTIONABILITY_TIER=N;PCGR_CSQ={_csq("intergenic_variant")}')) + _write_vcf(vcf_fp, v) + cancer_genes = tmp_path / 'genes.bed' + cancer_genes.write_text('chr1\t1\t9999999\n') + + orig_execute = util.execute_command + def fake_execute(cmd, **_): + import re + m = re.search(r'--output\s+(\S+)', cmd) + if m and 'bcftools annotate' in cmd: + shutil.copy(str(vcf_fp), m.group(1)) + else: + orig_execute(cmd) + + # limit=3: the 2 NONCODING category is dropped, 3 TIER_1 survive + with patch('bolt.common.constants.MAX_SOMATIC_VARIANTS', 3), \ + patch('bolt.util.execute_command', side_effect=fake_execute): + report_mod.select_pcgr_variants(vcf_fp, cancer_genes, 'TUMOR', tmp_path) + + filters_set_fp = tmp_path / 'TUMOR.pcgr_hypermutated.filters_set.vcf.gz' + self.assertTrue(filters_set_fp.exists(), 'filters_set VCF not created') + + all_records = list(cyvcf2.VCF(str(filters_set_fp))) + self.assertEqual(len(all_records), 5, 'filters_set VCF should contain all input variants') + + filter_tag = constants.VcfFilter.PCGR_COUNT_LIMIT.value + dropped = [r for r in all_records if filter_tag in (r.FILTERS or [])] + self.assertEqual(len(dropped), 2, 'Expected 2 NONCODING variants marked with PCGR_count_limit') + + +class TestGetImpacts(unittest.TestCase): + """Unit tests for pcgr.get_impacts() — CSQ string parsing.""" + + def test_single_consequence(self): + csq = _csq('intron_variant') + self.assertEqual(pcgr.get_impacts(csq), {'intron_variant'}) + + def test_multi_consequences_ampersand(self): + """A single CSQ entry with two consequences joined by & returns both.""" + csq = _csq('intron_variant&upstream_gene_variant') + self.assertEqual(pcgr.get_impacts(csq), {'intron_variant', 'upstream_gene_variant'}) + + def test_multiple_csq_entries_union(self): + """Comma-separated CSQ entries — returns the union of all consequences.""" + csq = f'{_csq("intron_variant")},{_csq("intergenic_variant")}' + self.assertEqual(pcgr.get_impacts(csq), {'intron_variant', 'intergenic_variant'}) + + +class TestDetermineFilter(unittest.TestCase): + """Unit tests for pcgr.determine_filter() — filter category determination.""" + + def _data(self, **overrides): + base = { + 'tier': None, + 'difficult': False, + 'giab_conf': False, + 'intergenic': None, + 'intronic': None, + 'downstream': None, + 'upstream': None, + 'impacts_other': None, + } + base.update(overrides) + return base + + def test_intergenic_difficult(self): + data = self._data(intergenic=True, difficult=True) + self.assertEqual(pcgr.determine_filter(data), ('intergenic', 'difficult')) + + def test_intergenic_no_region(self): + data = self._data(intergenic=True, difficult=False, giab_conf=False) + self.assertEqual(pcgr.determine_filter(data), ('intergenic', 'none')) + + def test_intergenic_giab_conf(self): + data = self._data(intergenic=True, giab_conf=True) + self.assertEqual(pcgr.determine_filter(data), ('intergenic', 'giab_conf')) + + def test_intronic_supersedes_intergenic(self): + """When both intergenic and intronic are present, intronic wins (higher priority).""" + data = self._data(intergenic=True, intronic=True, difficult=True) + self.assertEqual(pcgr.determine_filter(data), ('intronic', 'difficult')) + + def test_impacts_other_highest_priority(self): + """impacts_other is the last to be filtered — it wins over all other impacts.""" + data = self._data( + intergenic=True, intronic=True, downstream=True, + upstream=True, impacts_other=True, difficult=True, + ) + self.assertEqual(pcgr.determine_filter(data), ('impacts_other', 'difficult')) + + def test_no_impact_returns_false(self): + """A variant with no recognisable impact cannot be categorised.""" + data = self._data() # all impacts None + self.assertFalse(pcgr.determine_filter(data)) + + def test_giab_conf_region(self): + data = self._data(impacts_other=True, giab_conf=True) + self.assertEqual(pcgr.determine_filter(data), ('impacts_other', 'giab_conf')) + + +class TestGetVariantFilterData(unittest.TestCase): + """Unit tests for pcgr.get_variant_filter_data() — data extraction from VCF records.""" + + def test_tier_extracted(self): + info = f'PCGR_ACTIONABILITY_TIER=2;PCGR_CSQ={_csq("intron_variant")}' + data = pcgr.get_variant_filter_data(_make_variant(info)) + self.assertEqual(data['tier'], '2') + + def test_intergenic_impact(self): + info = f'PCGR_ACTIONABILITY_TIER=N;PCGR_CSQ={_csq("intergenic_variant")}' + data = pcgr.get_variant_filter_data(_make_variant(info)) + self.assertTrue(data['intergenic']) + self.assertFalse(data['intronic']) + self.assertFalse(data['downstream']) + self.assertFalse(data['upstream']) + self.assertFalse(data['impacts_other']) + + def test_intronic_impact(self): + info = f'PCGR_ACTIONABILITY_TIER=1;PCGR_CSQ={_csq("intron_variant")}' + data = pcgr.get_variant_filter_data(_make_variant(info)) + self.assertTrue(data['intronic']) + self.assertFalse(data['intergenic']) + + def test_giab_conf_overrides_difficult(self): + """GIAB_CONF flag must clear the difficult flag even when DIFFICULT_* is also present.""" + info = f'GIAB_CONF;DIFFICULT_segdup;PCGR_ACTIONABILITY_TIER=1;PCGR_CSQ={_csq("intron_variant")}' + data = pcgr.get_variant_filter_data(_make_variant(info)) + self.assertTrue(data['giab_conf']) + self.assertFalse(data['difficult']) + + def test_difficult_without_giab(self): + info = f'DIFFICULT_segdup;PCGR_ACTIONABILITY_TIER=1;PCGR_CSQ={_csq("intron_variant")}' + data = pcgr.get_variant_filter_data(_make_variant(info)) + self.assertTrue(data['difficult']) + self.assertFalse(data['giab_conf']) + + +if __name__ == '__main__': + unittest.main() From 5a28918c472cab19a593ea0328f00a39cb9e90e0 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Tue, 12 May 2026 15:16:18 +1000 Subject: [PATCH 40/89] add back source comment --- bolt/workflows/smlv_somatic/report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bolt/workflows/smlv_somatic/report.py b/bolt/workflows/smlv_somatic/report.py index a65efea..e181871 100644 --- a/bolt/workflows/smlv_somatic/report.py +++ b/bolt/workflows/smlv_somatic/report.py @@ -378,7 +378,7 @@ def select_pcgr_variants(vcf_fp, cancer_genes_fp, tumor_name, output_dir): fh_in = cyvcf2.VCF(vcf_fp) util.add_vcf_header_entry(fh_in, constants.VcfFilter.PCGR_COUNT_LIMIT) - # Additional VCF with all records for traceability + # NOTE(SW): creating an additional VCF with all records for traceability fp_out = output_dir / f'{tumor_name}.pcgr_hypermutated.pass.vcf.gz' fp_set_out = output_dir / f'{tumor_name}.pcgr_hypermutated.filters_set.vcf.gz' From 48488d6233df619f29d313a95ebbb4d60960d0b0 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Tue, 12 May 2026 15:23:57 +1000 Subject: [PATCH 41/89] DEV: install r-gpgr from input-sigrap-data branch for is_hypermutated testing Revert to conda pin r-gpgr ==2.3.0 before final release. --- docker/Dockerfile.gpgr | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/Dockerfile.gpgr b/docker/Dockerfile.gpgr index 6b76ecb..7464206 100644 --- a/docker/Dockerfile.gpgr +++ b/docker/Dockerfile.gpgr @@ -24,6 +24,11 @@ RUN \ 'bioconductor-txdb.hsapiens.ucsc.hg38.knowngene ==3.16.0' \ 'bioconductor-genomeinfodbdata ==1.2.9' +# DEV: override r-gpgr with branch version — revert to conda pin before release +RUN \ + conda run --prefix /env/ Rscript -e \ + "remotes::install_github('umccr/gpgr@input-sigrap-data', upgrade='never')" + COPY ./ /tmp/bolt/ RUN \ conda run --prefix /env/ pip install --no-deps /tmp/bolt/ From b97d21610283d7f34ad6381636e858bacc72b7b4 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Wed, 13 May 2026 13:56:51 +1000 Subject: [PATCH 42/89] Add tests for split_vcf chunking and document PCGR 500k handling - TestSplitVcf: verify split_vcf correctly chunks VCFs above MAX_SOMATIC_VARIANTS - Document PCGR 500k as a hard software limit (not OOM) in CLAUDE.md - Document known failure mode when tiered filtering cannot cap count (sash #52) --- CLAUDE.md | 68 +++++++++++++++++++++++++++++++++ tests/test_pcgr_hypermutated.py | 40 +++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..ed3ce6a --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,68 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Commands + +```bash +# Install locally (requires conda env with binary deps for full functionality) +pip install -e . + +# Run tests (canonical — matches CI) +python -m unittest discover tests/ --buffer + +# Run tests with pytest (also works) +python -m pytest tests/ -v + +# Run a single test +python -m pytest tests/test_smlv_somatic_filter.py::TestSmlvSomaticFilter::test_min_af_filter -v + +# Run bolt CLI directly +python -m bolt smlv_somatic filter --help +``` + +CI triggers: push to any branch runs tests; push of a `v*.*.*` tag builds and pushes all 6 Docker images to `ghcr.io/umccr/bolt`. + +## Architecture + +`bolt` is a Click CLI toolkit for UMCCR WGS somatic/germline post-processing. Entry point: `bolt/__main__.py` → `bolt/workflows/cli.py` auto-discovers workflow groups from `bolt/workflows/*/`. + +Each workflow directory (`smlv_somatic`, `smlv_germline`, `sv_somatic`, `other`) contains command modules. Each module exposes a `click` command named `entry`. Commands run as steps in the `sash` Nextflow pipeline, invoked via Docker images. + +**Six Docker images** split heavy dependencies — each command maps to exactly one image (see README). CI builds all six on `v*.*.*` tag push. + +## Key files + +| File | Role | +|------|------| +| `bolt/common/constants.py` | All thresholds, VCF tag enums (`VcfFilter`, `VcfInfo`, `VcfFormat`), and header definitions. Authoritative — add new tags here first. | +| `bolt/common/pcgr.py` | PCGR/CPSR invocation, VCF splitting/merging for hypermutated chunking, annotation transfer | +| `bolt/util.py` | `execute_command`, `command_prepare`, `add_vcf_header_entry`, VCF counting/merging helpers | +| `bolt/workflows/smlv_somatic/annotate.py` | PCGR annotation with chunking for hypermutated samples | +| `bolt/workflows/smlv_somatic/report.py` | PCGR cancer report, hypermutated variant selection (`select_pcgr_variants`) | +| `bolt/workflows/smlv_somatic/filter.py` | Somatic variant filtering — representative of the VCF processing pattern | + +## Coding patterns + +**CLI:** Each command uses `entry(ctx, **kwargs)` with `@click.option` decorators. Call `setup_logging(output_dir, script_name)` early in `entry`. + +**VCF writes:** Always use `cyvcf2.Writer(output_fp, in_fh, 'wz')` and close writers explicitly. Add new INFO/FILTER tags via `util.add_vcf_header_entry(fh, constants.VcfInfo.MY_TAG)` — never hardcode header strings. + +**Shell commands:** Use `util.execute_command(cmd)` for single commands. For pipelines where `pipefail` matters, wrap with `util.command_prepare(cmd)` first. + +## PCGR hypermutated handling + +PCGR enforces a hard 500k variant limit in software (not an OOM — it raises an explicit error). `MAX_SOMATIC_VARIANTS = 450_000` (`constants.py`) is the safe threshold used throughout bolt. + +- **Annotation path** (`annotate.py`): `split_vcf()` → chunks ≤450k → `run_somatic_chunk()` → merge. Handles any input size. ✅ Tested with synthetic 550k VCF — correctly splits into 450k + 100k chunks (2026-05-13). +- **Report path** (`report.py`): `select_pcgr_variants()` filters down to ≤450k via tiered priority (NONCODING first → TIER_1 last), then single `run_somatic()`. Requires `PCGR_ACTIONABILITY_TIER` (PCGR v2.2.1+) — if missing, variants have tier=None and fall into category `(None, impact, region)` which is absent from `get_ordering()`, so nothing gets dropped. If tiered filtering cannot reach ≤450k, raises `RuntimeError` — pipeline fails rather than hitting the PCGR hard software limit (not OOM). + - `PCGR_TIERS_FILTERING` uses short forms `('N','4','3','2','1')` matching values written by `transfer_annotations_somatic()`. +- `RETAIN_FIELDS_FILTERING` = `('PANEL', *HOTSPOT_FIELDS_FILTERING)` marks variants never dropped by tiered filtering. + +**Known failure mode (sash #52):** if a >500k sample has too few NONCODING/low-tier variants to drop, `select_pcgr_variants` raises `RuntimeError`. Fix not yet implemented — tracked in sash #52. + +## Release chain + +gpgr (conda publish) → `bolt/docker/Dockerfile.gpgr` (pin `r-gpgr ==X.Y.Z`) → bolt tag → 6 Docker images → sash module container tags. Runbook: `~/Documents/UMCCR/Runbooks/Release gpgr → bolt → sash.md`. + +Dev image pattern: install gpgr branch via `install_github` in `Dockerfile.gpgr` (marked `# DEV: … revert before release`). Build with `--platform linux/amd64` on Apple Silicon — `linux-aarch64` conda channel lacks `bcftools==1.17`. diff --git a/tests/test_pcgr_hypermutated.py b/tests/test_pcgr_hypermutated.py index 3dd786a..22c11c1 100644 --- a/tests/test_pcgr_hypermutated.py +++ b/tests/test_pcgr_hypermutated.py @@ -323,5 +323,45 @@ def test_difficult_without_giab(self): self.assertFalse(data['giab_conf']) +class TestSplitVcf(unittest.TestCase): + """Tests for pcgr.split_vcf() — chunking the annotation path for large VCFs. + + split_vcf() is the annotate-path strategy for hypermutated samples: it divides + a VCF into ≤MAX_SOMATIC_VARIANTS chunks so each chunk can be run through PCGR + independently. Tested 2026-05-13 with a synthetic 550k VCF: 550k → 450k + 100k. + """ + + def test_chunks_above_limit(self): + """VCF exceeding the limit is split into correctly-sized chunks.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + vcf_fp = tmp_path / 'input.vcf' + # 25 variants, limit=10 → expect 3 chunks (10, 10, 5) + v = [(i * 10, f'PCGR_CSQ={_csq("intron_variant")}') for i in range(1, 26)] + _write_vcf(vcf_fp, v) + + with patch('bolt.common.constants.MAX_SOMATIC_VARIANTS', 10): + chunks = pcgr.split_vcf(vcf_fp, tmp_path) + + self.assertEqual(len(chunks), 3) + counts = [_count_vcf(c) for c in chunks] + self.assertLessEqual(max(counts), 10) + self.assertEqual(sum(counts), 25) + + def test_no_chunking_within_limit(self): + """VCF within the limit produces a single chunk containing all variants.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + vcf_fp = tmp_path / 'input.vcf' + v = [(i * 10, f'PCGR_CSQ={_csq("intron_variant")}') for i in range(1, 6)] + _write_vcf(vcf_fp, v) + + with patch('bolt.common.constants.MAX_SOMATIC_VARIANTS', 10): + chunks = pcgr.split_vcf(vcf_fp, tmp_path) + + self.assertEqual(len(chunks), 1) + self.assertEqual(_count_vcf(chunks[0]), 5) + + if __name__ == '__main__': unittest.main() From cc397687690b1ef155990271e644fff4f6a998b2 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Wed, 13 May 2026 14:56:58 +1000 Subject: [PATCH 43/89] =?UTF-8?q?Fix=20positional=20arg=20shift=20in=20run?= =?UTF-8?q?=5Fsomatic=5Fchunk=20=E2=86=92=20run=5Fsomatic=20call?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit run_somatic has pcgr_threads between threads and pcgr_conda. The call in run_somatic_chunk used positional args and skipped pcgr_threads, causing pcgr_conda ('pcgr') to land in pcgr_threads → ValueError. Switch to keyword args matching the non-chunked call in annotate.py. Add regression test that mocks run_somatic and asserts pcgr_conda, pcgrr_conda, and threads are received at the correct parameters. --- bolt/common/pcgr.py | 2 +- tests/test_pcgr_hypermutated.py | 47 +++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/bolt/common/pcgr.py b/bolt/common/pcgr.py index a7bc9e2..e83687a 100644 --- a/bolt/common/pcgr.py +++ b/bolt/common/pcgr.py @@ -564,7 +564,7 @@ def run_somatic_chunk(vcf_chunks, pcgr_data_dir, vep_dir, output_dir, pcgr_outpu # Process each chunk sequentially for chunk_number, vcf_file in enumerate(vcf_chunks, start=1): - pcgr_tsv_fp, pcgr_vcf_fp = run_somatic(vcf_file, pcgr_data_dir, vep_dir, pcgr_output_dir, chunk_number, max_threads, pcgr_conda, pcgrr_conda) + pcgr_tsv_fp, pcgr_vcf_fp = run_somatic(vcf_file, pcgr_data_dir, vep_dir, pcgr_output_dir, chunk_nbr=chunk_number, threads=max_threads, pcgr_conda=pcgr_conda, pcgrr_conda=pcgrr_conda) if pcgr_tsv_fp: pcgr_tsv_files.append(pcgr_tsv_fp) if pcgr_vcf_fp: diff --git a/tests/test_pcgr_hypermutated.py b/tests/test_pcgr_hypermutated.py index 22c11c1..73ab9e7 100644 --- a/tests/test_pcgr_hypermutated.py +++ b/tests/test_pcgr_hypermutated.py @@ -363,5 +363,52 @@ def test_no_chunking_within_limit(self): self.assertEqual(_count_vcf(chunks[0]), 5) +class TestRunSomaticChunkArgMapping(unittest.TestCase): + """Regression test: run_somatic_chunk must forward args as keywords to run_somatic. + + Before the fix, run_somatic_chunk called run_somatic positionally (6 args), + skipping pcgr_threads. This caused pcgr_conda ('pcgr') to land in the + pcgr_threads slot → ValueError: invalid literal for int() with base 10: 'pcgr'. + """ + + def test_pcgr_conda_not_shifted_into_pcgr_threads(self): + """pcgr_conda must reach run_somatic as pcgr_conda, not as pcgr_threads.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + vcf_fp = tmp_path / 'chunk.vcf' + _write_vcf(vcf_fp, [(10, f'PCGR_CSQ={_csq("intron_variant")}')]) + + captured = {} + + def fake_run_somatic(*args, **kwargs): + captured['args'] = args + captured['kwargs'] = kwargs + return (None, None) + + with patch('bolt.common.pcgr.run_somatic', side_effect=fake_run_somatic), \ + patch('bolt.common.pcgr.merging_pcgr_files', + return_value=(tmp_path / 'out.vcf', tmp_path / 'out.tsv')): + pcgr.run_somatic_chunk( + [vcf_fp], + pcgr_data_dir=tmp_path / 'pcgr_data', + vep_dir=tmp_path / 'vep', + output_dir=tmp_path, + pcgr_output_dir=tmp_path / 'pcgr_output', + max_threads=4, + pcgr_conda='pcgr_env', + pcgrr_conda='pcgrr_env', + ) + + kw = captured['kwargs'] + self.assertEqual(kw.get('pcgr_conda'), 'pcgr_env', + 'pcgr_conda was not forwarded — likely shifted into pcgr_threads') + self.assertEqual(kw.get('pcgrr_conda'), 'pcgrr_env', + 'pcgrr_conda was not forwarded correctly') + self.assertEqual(kw.get('threads'), 4, + 'threads (max_threads) was not forwarded correctly') + self.assertEqual(kw.get('chunk_nbr'), 1, + 'chunk_nbr was not forwarded correctly') + + if __name__ == '__main__': unittest.main() From 7f0100109f140e43e913c75efe96adfc5434f046 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 15 May 2026 09:30:57 +1000 Subject: [PATCH 44/89] fix hypermutated tag --- bolt/workflows/smlv_somatic/report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bolt/workflows/smlv_somatic/report.py b/bolt/workflows/smlv_somatic/report.py index e181871..66c43c0 100644 --- a/bolt/workflows/smlv_somatic/report.py +++ b/bolt/workflows/smlv_somatic/report.py @@ -301,7 +301,7 @@ def count_variant_process(vcf_fp): if not record.FILTER or rescued_filters: counts['filter_pass'] += 1 - counts['is_hypermutated'] = counts['filter_pass'] > constants.MAX_SOMATIC_VARIANTS + counts['is_hypermutated'] = counts['dragen'] > constants.MAX_SOMATIC_VARIANTS return counts From 97ad9f76fc5636867a208a0974540616f270b320 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 15 May 2026 09:48:26 +1000 Subject: [PATCH 45/89] Add tests for is_hypermutated dragen count fix Covers count_variant_process: dragen vs filter_pass threshold, SAGE_NOVEL exclusion, and annotation filter exclusion. --- tests/test_pcgr_hypermutated.py | 95 +++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/tests/test_pcgr_hypermutated.py b/tests/test_pcgr_hypermutated.py index 73ab9e7..edbadb1 100644 --- a/tests/test_pcgr_hypermutated.py +++ b/tests/test_pcgr_hypermutated.py @@ -410,5 +410,100 @@ def fake_run_somatic(*args, **kwargs): 'chunk_nbr was not forwarded correctly') +class TestCountVariantProcess(unittest.TestCase): + """Verify count_variant_process counts and is_hypermutated flag (bolt #27). + + is_hypermutated must use the 'dragen' count (raw, pre-bolt-filter), not + 'filter_pass'. A sample with many DRAGEN variants that are mostly filtered + away must still be flagged as hypermutated. + """ + + # Minimal header for count_variant_process: needs FILTER tags + SAGE_NOVEL INFO + COUNT_HEADER = ( + '##fileformat=VCFv4.2\n' + '##FILTER=\n' + f'##FILTER=\n' + f'##FILTER=\n' + f'##FILTER=\n' + f'##INFO=\n' + f'##INFO=\n' + '##contig=\n' + '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n' + ) + + def _write_count_vcf(self, path, rows): + """rows: list of (pos, filter_str, info_str) tuples.""" + with open(path, 'w') as fh: + fh.write(self.COUNT_HEADER) + for pos, filt, info in rows: + fh.write(f'chr1\t{pos}\t.\tA\tT\t.\t{filt}\t{info}\n') + + def test_is_hypermutated_uses_dragen_count(self): + """is_hypermutated=True when dragen count > MAX_SOMATIC_VARIANTS even if filter_pass is below.""" + with tempfile.TemporaryDirectory() as tmp: + vcf_fp = pathlib.Path(tmp) / 'test.vcf' + min_af = constants.VcfFilter.MIN_AF.value + # 3 DRAGEN PASS variants + 2 filtered by bolt (MIN_AF) — filter_pass=3, dragen=5 + rows = [(i * 10, 'PASS', '.') for i in range(1, 4)] + rows += [(i * 10 + 5, min_af, '.') for i in range(1, 3)] + self._write_count_vcf(vcf_fp, rows) + + with patch('bolt.common.constants.MAX_SOMATIC_VARIANTS', 4): + counts = report_mod.count_variant_process(vcf_fp) + + self.assertEqual(counts['dragen'], 5) + self.assertEqual(counts['filter_pass'], 3) + # dragen(5) > MAX(4) → hypermutated, even though filter_pass(3) ≤ MAX(4) + self.assertTrue(counts['is_hypermutated']) + + def test_is_hypermutated_false_when_dragen_within_limit(self): + """is_hypermutated=False when dragen count ≤ MAX_SOMATIC_VARIANTS.""" + with tempfile.TemporaryDirectory() as tmp: + vcf_fp = pathlib.Path(tmp) / 'test.vcf' + rows = [(i * 10, 'PASS', '.') for i in range(1, 4)] + self._write_count_vcf(vcf_fp, rows) + + with patch('bolt.common.constants.MAX_SOMATIC_VARIANTS', 10): + counts = report_mod.count_variant_process(vcf_fp) + + self.assertEqual(counts['dragen'], 3) + self.assertFalse(counts['is_hypermutated']) + + def test_sage_novel_excluded_from_dragen_count(self): + """SAGE_NOVEL variants are not counted as DRAGEN variants.""" + with tempfile.TemporaryDirectory() as tmp: + vcf_fp = pathlib.Path(tmp) / 'test.vcf' + sage_novel_info = constants.VcfInfo.SAGE_NOVEL.value + rows = [ + (10, 'PASS', '.'), # dragen + (20, 'PASS', sage_novel_info), # sage novel — not dragen + (30, 'PASS', '.'), # dragen + ] + self._write_count_vcf(vcf_fp, rows) + + with patch('bolt.common.constants.MAX_SOMATIC_VARIANTS', 100): + counts = report_mod.count_variant_process(vcf_fp) + + self.assertEqual(counts['dragen'], 2) + self.assertEqual(counts['sage'], 3) + + def test_annotation_filter_excluded_from_annotated_count(self): + """Variants with bolt annotation filters are excluded from annotated count.""" + with tempfile.TemporaryDirectory() as tmp: + vcf_fp = pathlib.Path(tmp) / 'test.vcf' + annot_filter = constants.VcfFilter.MAX_VARIANTS_NON_PASS.value + rows = [ + (10, 'PASS', '.'), # annotated + (20, annot_filter, '.'), # not annotated (bolt annotation filter) + ] + self._write_count_vcf(vcf_fp, rows) + + with patch('bolt.common.constants.MAX_SOMATIC_VARIANTS', 100): + counts = report_mod.count_variant_process(vcf_fp) + + self.assertEqual(counts['annotated'], 1) + self.assertEqual(counts['dragen'], 2) + + if __name__ == '__main__': unittest.main() From b4ab71f44258de7e35445cbca2b63170c5a6e17c Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 15 May 2026 09:48:30 +1000 Subject: [PATCH 46/89] Separate stdout/stderr streams in execute_command Uses select() to multiplex both pipes in one thread, preventing pipe buffer deadlock and routing stderr to WARNING level. --- bolt/util.py | 85 ++++++++++++++++++++++++++++------------------------ 1 file changed, 46 insertions(+), 39 deletions(-) diff --git a/bolt/util.py b/bolt/util.py index bd89c13..c130c73 100644 --- a/bolt/util.py +++ b/bolt/util.py @@ -1,5 +1,6 @@ import gzip import pathlib +import select import subprocess import sys import textwrap @@ -22,63 +23,69 @@ def get_project_root(): def execute_command(command, log_file_path=None): - # Wrap command with proper error handling # set -e: exit on error, -u: exit on unset variable, -o pipefail: pipeline fails if any command fails prepared_command = f'set -euo pipefail; {textwrap.dedent(command)}' - logger.info("Executing command: %s", command.strip()) - # Open the log file if provided - log_file = log_file_path.open('a', encoding='utf-8') if log_file_path else None - - # Prepare environment for subprocess (inherit current environment) - env = subprocess.os.environ.copy() - - # Launch process with combined stdout and stderr streams, and line buffering enabled. process = subprocess.Popen( prepared_command, shell=True, executable='/bin/bash', stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, + stderr=subprocess.PIPE, text=True, encoding='utf-8', bufsize=1, # line buffered - env=env ) - output_lines = [] - # Iterate over each line as it becomes available - with process.stdout: - for line in iter(process.stdout.readline, ''): - if line: - # Filter out bash libtinfo.so.6 warnings - if 'libtinfo.so.6: no version information available' not in line: - logger.info(line.strip()) - output_lines.append(line) - if log_file: - log_file.write(line) - log_file.flush() # flush immediately for real-time logging - process.wait() # wait for the process to complete - - if log_file: - log_file.close() - - result = SimpleNamespace( - stdout=''.join(output_lines), + stdout_lines = [] + stderr_lines = [] + stream_map = { + process.stdout: (stdout_lines, logger.info), + process.stderr: (stderr_lines, logger.warning), + } + open_streams = set(stream_map) + log_file = log_file_path.open('a', encoding='utf-8') if log_file_path else None + + try: + # select multiplexes stdout and stderr in a single thread, preserving arrival order + # and preventing pipe buffer deadlock without threading races on log_file writes + while open_streams: + readable, _, _ = select.select(open_streams, [], []) + for stream in readable: + line = stream.readline() + if line: + # Filter out bash libtinfo.so.6 warnings + if 'libtinfo.so.6: no version information available' not in line: + lines, log_fn = stream_map[stream] + log_fn(line.rstrip()) + lines.append(line) + if log_file: + log_file.write(line) + log_file.flush() + else: + open_streams.discard(stream) + finally: + process.wait() + if log_file: + log_file.close() + + if process.returncode != 0: + logger.error("Command failed with return code %d: %s", process.returncode, command.strip()) + raise subprocess.CalledProcessError( + process.returncode, command, + output=''.join(stdout_lines), + stderr=''.join(stderr_lines), + ) + + return SimpleNamespace( + stdout=''.join(stdout_lines), + stderr=''.join(stderr_lines), returncode=process.returncode, pid=process.pid, - command=command + command=command, ) - # Raise exception on non-zero return code - if result.returncode != 0: - error_msg = f"Command failed with return code {result.returncode}: {command.strip()}" - logger.error(error_msg) - raise subprocess.CalledProcessError(result.returncode, command, output=''.join(output_lines)) - - return result - def count_vcf_records(fp): result = execute_command(f'bcftools view -H {fp} | wc -l') return int(result.stdout.strip()) From 1b366fa6cd17bcef2472e1d9829b1b54c5643a44 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 15 May 2026 10:09:41 +1000 Subject: [PATCH 47/89] Pin r-gpgr ==2.3.0 in Dockerfile.gpgr Includes hypermutated flag fix (reads is_hypermutated from bolt JSON) and SNVs display cleanup. --- docker/Dockerfile.gpgr | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/docker/Dockerfile.gpgr b/docker/Dockerfile.gpgr index 7464206..950391a 100644 --- a/docker/Dockerfile.gpgr +++ b/docker/Dockerfile.gpgr @@ -18,17 +18,12 @@ RUN \ RUN \ conda install --prefix /env/ \ - 'r-gpgr ==2.2.11' \ + 'r-gpgr ==2.3.0' \ 'r-sigrap ==0.1.1' \ 'bioconductor-bsgenome.hsapiens.ucsc.hg38 ==1.4.5' \ 'bioconductor-txdb.hsapiens.ucsc.hg38.knowngene ==3.16.0' \ 'bioconductor-genomeinfodbdata ==1.2.9' -# DEV: override r-gpgr with branch version — revert to conda pin before release -RUN \ - conda run --prefix /env/ Rscript -e \ - "remotes::install_github('umccr/gpgr@input-sigrap-data', upgrade='never')" - COPY ./ /tmp/bolt/ RUN \ conda run --prefix /env/ pip install --no-deps /tmp/bolt/ From f77deb410f7061d9541658060b493e5697410242 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 15 May 2026 10:29:39 +1000 Subject: [PATCH 48/89] fix doc and missing file closure --- bolt/common/constants.py | 6 +++--- bolt/common/pcgr.py | 10 ++++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/bolt/common/constants.py b/bolt/common/constants.py index 0dba0b9..d4b52b5 100644 --- a/bolt/common/constants.py +++ b/bolt/common/constants.py @@ -277,9 +277,9 @@ def namespace(self): 'Number': '1', 'Type': 'String', 'Description': ( - 'Tier reported by PCGR with the following meaning: TIER_1: strong clinical ' - 'significance; TIER_2: potential clinical significance; TIER_3: uncertain clinical ' - 'significance; TIER_4: other coding variants; NONCODING: other non-coding variants' + 'Tier reported by PCGR: 1: strong clinical significance; ' + '2: potential clinical significance; 3: uncertain clinical significance; ' + '4: other coding variants; N: other non-coding variants' ), }, VcfInfo.PCGR_CSQ: { diff --git a/bolt/common/pcgr.py b/bolt/common/pcgr.py index e83687a..6f2d6ca 100644 --- a/bolt/common/pcgr.py +++ b/bolt/common/pcgr.py @@ -312,6 +312,7 @@ def transfer_annotations_somatic(input_fp, tumor_name, pcgr_vcf_fp, pcgr_tsv_fp, # Annotate and write record_ann = annotate_record(record, pcgr_data, allow_missing=True) output_fh.write_record(record_ann) + output_fh.close() def transfer_annotations_germline(input_fp, normal_name, cpsr_dir, output_dir): @@ -349,6 +350,7 @@ def transfer_annotations_germline(input_fp, normal_name, cpsr_dir, output_dir): # NOTE(SW): allow missing CPSR annotations for input variants, CPSR seems to drop some record_ann = annotate_record(record, cpsr_data, allow_missing=True) output_fh.write_record(record_ann) + output_fh.close() def collect_pcgr_annotation_data(tsv_fp, vcf_fp, info_field_map): @@ -530,12 +532,12 @@ def split_vcf(input_vcf, output_dir, *, max_variants=None): variant_count = 0 input_vcf = pathlib.Path(input_vcf) base_filename = input_vcf.stem - chunk_filename = output_dir / f"{base_filename}_chunk{chunk_number}.vcf" + chunk_filename = output_dir / f"{base_filename}_chunk{chunk_number}.vcf.gz" chunk_files.append(chunk_filename) # Open the input VCF using cyvcf2 vcf_in = cyvcf2.VCF(input_vcf) # Create a new VCF file for the first chunk - vcf_out = cyvcf2.Writer(str(chunk_filename), vcf_in) + vcf_out = cyvcf2.Writer(str(chunk_filename), vcf_in, 'wz') last_position = None for record in vcf_in: current_position = record.POS @@ -544,9 +546,9 @@ def split_vcf(input_vcf, output_dir, *, max_variants=None): # Close the current chunk file and start a new one vcf_out.close() chunk_number += 1 - chunk_filename = output_dir / f"{base_filename}_chunk{chunk_number}.vcf" + chunk_filename = output_dir / f"{base_filename}_chunk{chunk_number}.vcf.gz" chunk_files.append(chunk_filename) - vcf_out = cyvcf2.Writer(str(chunk_filename), vcf_in) + vcf_out = cyvcf2.Writer(str(chunk_filename), vcf_in, 'wz') variant_count = 0 # Write the record to the current chunk vcf_out.write_record(record) From 1b6e28a78a96dcb1a9ed053f2a00a7ed85475f44 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 15 May 2026 11:33:51 +1000 Subject: [PATCH 49/89] add test --- tests/test_pcgr_hypermutated.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_pcgr_hypermutated.py b/tests/test_pcgr_hypermutated.py index edbadb1..962b9ad 100644 --- a/tests/test_pcgr_hypermutated.py +++ b/tests/test_pcgr_hypermutated.py @@ -362,6 +362,21 @@ def test_no_chunking_within_limit(self): self.assertEqual(len(chunks), 1) self.assertEqual(_count_vcf(chunks[0]), 5) + def test_chunks_are_gzipped(self): + """Chunk files must be .vcf.gz — plain .vcf chunks violate CLAUDE.md and waste disk.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + vcf_fp = tmp_path / 'input.vcf' + v = [(i * 10, f'PCGR_CSQ={_csq("intron_variant")}') for i in range(1, 26)] + _write_vcf(vcf_fp, v) + + with patch('bolt.common.constants.MAX_SOMATIC_VARIANTS', 10): + chunks = pcgr.split_vcf(vcf_fp, tmp_path) + + for chunk in chunks: + self.assertTrue(str(chunk).endswith('.vcf.gz'), + f'Expected .vcf.gz chunk, got: {chunk.name}') + class TestRunSomaticChunkArgMapping(unittest.TestCase): """Regression test: run_somatic_chunk must forward args as keywords to run_somatic. From 58987d1bce3935cefdf450f6c849254414a3047c Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 15 May 2026 11:34:29 +1000 Subject: [PATCH 50/89] change .with_suffix : Path.with_suffix() only replaces the last suffix everything after the final dot. The caller passes a path ending in .pass --- bolt/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bolt/util.py b/bolt/util.py index c130c73..c3b07fe 100644 --- a/bolt/util.py +++ b/bolt/util.py @@ -156,8 +156,8 @@ def merge_vcf_files(vcf_files, merged_vcf_fp): - Path to the sorted merged VCF file. """ merged_vcf_fp = pathlib.Path(merged_vcf_fp) - merged_unsorted_vcf = merged_vcf_fp.with_suffix('.unsorted.vcf.gz') - merged_vcf = merged_vcf_fp.with_suffix('.vcf.gz') + merged_unsorted_vcf = merged_vcf_fp.parent / f'{merged_vcf_fp.name}.unsorted.vcf.gz' + merged_vcf = merged_vcf_fp.parent / f'{merged_vcf_fp.name}.vcf.gz' # Prepare the bcftools merge command arguments command_args = [ From 59495397eaaf71d9af75e6dc0d2e5c4dc7944623 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 15 May 2026 14:05:27 +1000 Subject: [PATCH 51/89] chore: untrack local-only files --- CLAUDE.md | 68 ------------------------------------------------------- 1 file changed, 68 deletions(-) delete mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index ed3ce6a..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,68 +0,0 @@ -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -## Commands - -```bash -# Install locally (requires conda env with binary deps for full functionality) -pip install -e . - -# Run tests (canonical — matches CI) -python -m unittest discover tests/ --buffer - -# Run tests with pytest (also works) -python -m pytest tests/ -v - -# Run a single test -python -m pytest tests/test_smlv_somatic_filter.py::TestSmlvSomaticFilter::test_min_af_filter -v - -# Run bolt CLI directly -python -m bolt smlv_somatic filter --help -``` - -CI triggers: push to any branch runs tests; push of a `v*.*.*` tag builds and pushes all 6 Docker images to `ghcr.io/umccr/bolt`. - -## Architecture - -`bolt` is a Click CLI toolkit for UMCCR WGS somatic/germline post-processing. Entry point: `bolt/__main__.py` → `bolt/workflows/cli.py` auto-discovers workflow groups from `bolt/workflows/*/`. - -Each workflow directory (`smlv_somatic`, `smlv_germline`, `sv_somatic`, `other`) contains command modules. Each module exposes a `click` command named `entry`. Commands run as steps in the `sash` Nextflow pipeline, invoked via Docker images. - -**Six Docker images** split heavy dependencies — each command maps to exactly one image (see README). CI builds all six on `v*.*.*` tag push. - -## Key files - -| File | Role | -|------|------| -| `bolt/common/constants.py` | All thresholds, VCF tag enums (`VcfFilter`, `VcfInfo`, `VcfFormat`), and header definitions. Authoritative — add new tags here first. | -| `bolt/common/pcgr.py` | PCGR/CPSR invocation, VCF splitting/merging for hypermutated chunking, annotation transfer | -| `bolt/util.py` | `execute_command`, `command_prepare`, `add_vcf_header_entry`, VCF counting/merging helpers | -| `bolt/workflows/smlv_somatic/annotate.py` | PCGR annotation with chunking for hypermutated samples | -| `bolt/workflows/smlv_somatic/report.py` | PCGR cancer report, hypermutated variant selection (`select_pcgr_variants`) | -| `bolt/workflows/smlv_somatic/filter.py` | Somatic variant filtering — representative of the VCF processing pattern | - -## Coding patterns - -**CLI:** Each command uses `entry(ctx, **kwargs)` with `@click.option` decorators. Call `setup_logging(output_dir, script_name)` early in `entry`. - -**VCF writes:** Always use `cyvcf2.Writer(output_fp, in_fh, 'wz')` and close writers explicitly. Add new INFO/FILTER tags via `util.add_vcf_header_entry(fh, constants.VcfInfo.MY_TAG)` — never hardcode header strings. - -**Shell commands:** Use `util.execute_command(cmd)` for single commands. For pipelines where `pipefail` matters, wrap with `util.command_prepare(cmd)` first. - -## PCGR hypermutated handling - -PCGR enforces a hard 500k variant limit in software (not an OOM — it raises an explicit error). `MAX_SOMATIC_VARIANTS = 450_000` (`constants.py`) is the safe threshold used throughout bolt. - -- **Annotation path** (`annotate.py`): `split_vcf()` → chunks ≤450k → `run_somatic_chunk()` → merge. Handles any input size. ✅ Tested with synthetic 550k VCF — correctly splits into 450k + 100k chunks (2026-05-13). -- **Report path** (`report.py`): `select_pcgr_variants()` filters down to ≤450k via tiered priority (NONCODING first → TIER_1 last), then single `run_somatic()`. Requires `PCGR_ACTIONABILITY_TIER` (PCGR v2.2.1+) — if missing, variants have tier=None and fall into category `(None, impact, region)` which is absent from `get_ordering()`, so nothing gets dropped. If tiered filtering cannot reach ≤450k, raises `RuntimeError` — pipeline fails rather than hitting the PCGR hard software limit (not OOM). - - `PCGR_TIERS_FILTERING` uses short forms `('N','4','3','2','1')` matching values written by `transfer_annotations_somatic()`. -- `RETAIN_FIELDS_FILTERING` = `('PANEL', *HOTSPOT_FIELDS_FILTERING)` marks variants never dropped by tiered filtering. - -**Known failure mode (sash #52):** if a >500k sample has too few NONCODING/low-tier variants to drop, `select_pcgr_variants` raises `RuntimeError`. Fix not yet implemented — tracked in sash #52. - -## Release chain - -gpgr (conda publish) → `bolt/docker/Dockerfile.gpgr` (pin `r-gpgr ==X.Y.Z`) → bolt tag → 6 Docker images → sash module container tags. Runbook: `~/Documents/UMCCR/Runbooks/Release gpgr → bolt → sash.md`. - -Dev image pattern: install gpgr branch via `install_github` in `Dockerfile.gpgr` (marked `# DEV: … revert before release`). Build with `--platform linux/amd64` on Apple Silicon — `linux-aarch64` conda channel lacks `bcftools==1.17`. From e0103ef79154ab740e5ca6284745e721bdda13fb Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 15 May 2026 15:17:21 +1000 Subject: [PATCH 52/89] update CHANGELOG for 0.3.0 release --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f026d5a..48c4531 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # bolt changelog -## dev +## 0.3.0 - [17](https://github.com/umccr/bolt/pull/17) - change dragen HRD file optional From 149db35ce9316af6b130887bbb25b8268298b546 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Mon, 18 May 2026 11:12:15 +1000 Subject: [PATCH 53/89] bump version to 0.3.1 and update changelog --- .bumpversion.cfg | 2 +- CHANGELOG.md | 11 ++++++++++- conda/env/bolt_env.yml | 1 + pyproject.toml | 2 +- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 4bae614..d86a437 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.0 +current_version = 0.3.1 commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P[a-z0-9+]+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48c4531..9eadff9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # bolt changelog +## 0.3.1 + +- Fix `ModuleNotFoundError: No module named 'pkg_resources'` in `bolt:0.3.0-multiqc` — add `setuptools <81` to conda env +- Fix `merge_vcf_files` producing wrong output filename — `Path.with_suffix()` was stripping `.pass` component; use explicit path concatenation instead +- Fix VCF writers not closed in `transfer_annotations_somatic` and `transfer_annotations_germline` — BGZip output could be truncated +- Fix `split_vcf` writing uncompressed plain `.vcf` chunks — now uses `.vcf.gz` with `wz` mode +- Fix `PCGR_ACTIONABILITY_TIER` VCF header description — updated to match stored short-form values (`1`,`2`,`3`,`4`,`N`) +- Add regression test for chunk file compression (`test_chunks_are_gzipped`) + ## 0.3.0 - [17](https://github.com/umccr/bolt/pull/17) - change dragen HRD file optional @@ -10,4 +19,4 @@ - [6](https://github.com/umccr/bolt/pull/6) - Change oncoanalyser v2.0.0 update, with switch sv caller from GRIPSS to eSVee -- [9](https://github.com/umccr/bolt/pull/9) Add hypermutation sample handling \ No newline at end of file +- [9](https://github.com/umccr/bolt/pull/9) Add hypermutation sample handling diff --git a/conda/env/bolt_env.yml b/conda/env/bolt_env.yml index 71e90c9..fe7200b 100644 --- a/conda/env/bolt_env.yml +++ b/conda/env/bolt_env.yml @@ -15,5 +15,6 @@ dependencies: - pybedtools - python >=3.10 - pyyaml + - setuptools <81 - vcfanno ==0.3.5 - ncurses>=6.3 diff --git a/pyproject.toml b/pyproject.toml index b9c2128..6dd2fba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ include = ["bolt*"] [project] name = "bolt" -version = "0.3.0" +version = "0.3.1" authors = [ {name = "Stephen Watts", email = "stephen.watts@umccr.org"}, ] From 358290a95b7ff74098c2c9eb1d8759d4c4aaee46 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Mon, 18 May 2026 11:15:00 +1000 Subject: [PATCH 54/89] ci: add smoke tests to catch image startup failures before push --- .github/workflows/build.yaml | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8e40ae2..9009f9c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -23,7 +23,7 @@ jobs: # this might remove tools that are actually needed, # if set to "true" but frees about 6 GB tool-cache: false - + # all of these default to true, but feel free to set to # "false" if necessary for your workflow android: true @@ -32,6 +32,7 @@ jobs: large-packages: true docker-images: true swap-storage: true + - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: @@ -42,7 +43,8 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Build and push image + - name: Build image + id: build run: | if [[ "${{ matrix.build_name }}" == "base" ]]; then docker_tag=bolt:${GITHUB_REF_NAME#v}; @@ -52,10 +54,26 @@ jobs: dockerfile_fn=Dockerfile.${{ matrix.build_name }}; fi; - # Build and push to GHCR umccr registry docker build \ --platform linux/amd64 \ + --load \ -f docker/${dockerfile_fn} \ -t ghcr.io/umccr/${docker_tag} \ - --push \ .; + + echo "docker_tag=ghcr.io/umccr/${docker_tag}" >> $GITHUB_OUTPUT + + - name: Smoke test + run: | + docker run --rm ${{ steps.build.outputs.docker_tag }} bolt --version + + case "${{ matrix.build_name }}" in + multiqc) docker run --rm ${{ steps.build.outputs.docker_tag }} multiqc --version ;; + gpgr) docker run --rm ${{ steps.build.outputs.docker_tag }} Rscript -e 'library(gpgr); cat("gpgr ok\n")' ;; + pcgr) docker run --rm ${{ steps.build.outputs.docker_tag }} pcgr --version ;; + snpeff) docker run --rm ${{ steps.build.outputs.docker_tag }} bash -c 'snpEff -version 2>&1 | head -1' ;; + circos) docker run --rm ${{ steps.build.outputs.docker_tag }} circos --version ;; + esac + + - name: Push image + run: docker push ${{ steps.build.outputs.docker_tag }} From dae66e5439798254d6be3d8ff00cc073a50f9044 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Tue, 19 May 2026 11:24:44 +1000 Subject: [PATCH 55/89] fix missing build and pcgr docker file --- .github/workflows/build.yaml | 2 +- docker/Dockerfile.pcgr | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9009f9c..22018a8 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -70,7 +70,7 @@ jobs: case "${{ matrix.build_name }}" in multiqc) docker run --rm ${{ steps.build.outputs.docker_tag }} multiqc --version ;; gpgr) docker run --rm ${{ steps.build.outputs.docker_tag }} Rscript -e 'library(gpgr); cat("gpgr ok\n")' ;; - pcgr) docker run --rm ${{ steps.build.outputs.docker_tag }} pcgr --version ;; + pcgr) docker run --rm ${{ steps.build.outputs.docker_tag }} conda run --no-capture-output -n pcgr pcgr --version ;; snpeff) docker run --rm ${{ steps.build.outputs.docker_tag }} bash -c 'snpEff -version 2>&1 | head -1' ;; circos) docker run --rm ${{ steps.build.outputs.docker_tag }} circos --version ;; esac diff --git a/docker/Dockerfile.pcgr b/docker/Dockerfile.pcgr index 2619305..c63e694 100644 --- a/docker/Dockerfile.pcgr +++ b/docker/Dockerfile.pcgr @@ -44,5 +44,5 @@ FROM quay.io/bioconda/base-glibc-busybox-bash:2.1.0 # Copy Conda install and all environments COPY --from=build /opt/conda/ /opt/conda/ -ENV PATH="/opt/conda/envs/bolt/bin:/opt/conda/bin:${PATH}" +ENV PATH="/opt/conda/envs/bolt/bin:/opt/conda/envs/pcgr/bin:/opt/conda/bin:${PATH}" ENV LD_LIBRARY_PATH="/opt/conda/lib/" From c53557a6e9e484eb17ab286d2f8aead4afc52ac5 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Tue, 19 May 2026 11:27:22 +1000 Subject: [PATCH 56/89] Fix remove unused package and log pcgr limit --- bolt/common/constants.py | 2 +- bolt/workflows/other/cancer_report.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/bolt/common/constants.py b/bolt/common/constants.py index d4b52b5..3043f74 100644 --- a/bolt/common/constants.py +++ b/bolt/common/constants.py @@ -231,7 +231,7 @@ def namespace(self): VcfFilter.PCGR_COUNT_LIMIT: { 'Description': ( f'Manually filtered to {MAX_SOMATIC_VARIANTS} variants to stay below the ' - 'PCGR 500,000 variant limit' + f'PCGR hard limit of 500,000 variants' ), }, diff --git a/bolt/workflows/other/cancer_report.py b/bolt/workflows/other/cancer_report.py index 7e2d70f..ebd2f3a 100644 --- a/bolt/workflows/other/cancer_report.py +++ b/bolt/workflows/other/cancer_report.py @@ -2,7 +2,6 @@ import click -import logging from ... import util From 8a35851ecb6134d5df79266b42064ee2352f457a Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Tue, 19 May 2026 13:46:36 +1000 Subject: [PATCH 57/89] fix: tabix-index VCF chunks after split_vcf so PCGR can read them --- bolt/common/pcgr.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bolt/common/pcgr.py b/bolt/common/pcgr.py index 6f2d6ca..fc2151c 100644 --- a/bolt/common/pcgr.py +++ b/bolt/common/pcgr.py @@ -557,6 +557,10 @@ def split_vcf(input_vcf, output_dir, *, max_variants=None): # Close the last chunk file vcf_out.close() vcf_in.close() + + for chunk_fp in chunk_files: + util.execute_command(f'bcftools index --tbi {chunk_fp}') + logger.info(f"VCF file split into {len(chunk_files)} chunks.") return chunk_files From cfac848ae0f97bc61264052034fff71d4c635894 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Tue, 19 May 2026 14:03:02 +1000 Subject: [PATCH 58/89] test: assert split_vcf chunks have .tbi index (regression for PCGR v2.2.5 tabix check) --- tests/test_pcgr_hypermutated.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_pcgr_hypermutated.py b/tests/test_pcgr_hypermutated.py index 962b9ad..93e4390 100644 --- a/tests/test_pcgr_hypermutated.py +++ b/tests/test_pcgr_hypermutated.py @@ -377,6 +377,21 @@ def test_chunks_are_gzipped(self): self.assertTrue(str(chunk).endswith('.vcf.gz'), f'Expected .vcf.gz chunk, got: {chunk.name}') + def test_chunks_are_tabix_indexed(self): + """Each .vcf.gz chunk must have a .tbi index — PCGR v2.2.5 requires it.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + vcf_fp = tmp_path / 'input.vcf' + v = [(i * 10, f'PCGR_CSQ={_csq("intron_variant")}') for i in range(1, 26)] + _write_vcf(vcf_fp, v) + + with patch('bolt.common.constants.MAX_SOMATIC_VARIANTS', 10): + chunks = pcgr.split_vcf(vcf_fp, tmp_path) + + for chunk in chunks: + tbi = pathlib.Path(str(chunk) + '.tbi') + self.assertTrue(tbi.exists(), f'Missing tabix index for {chunk.name}') + class TestRunSomaticChunkArgMapping(unittest.TestCase): """Regression test: run_somatic_chunk must forward args as keywords to run_somatic. From d2c390ff5ac68dcb660b092e8099b01e103c454d Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Sun, 24 May 2026 22:51:10 +1000 Subject: [PATCH 59/89] fix: bump r-gpgr to 2.3.1 in gpgr Docker image --- docker/Dockerfile.gpgr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.gpgr b/docker/Dockerfile.gpgr index 950391a..d22d5ec 100644 --- a/docker/Dockerfile.gpgr +++ b/docker/Dockerfile.gpgr @@ -18,7 +18,7 @@ RUN \ RUN \ conda install --prefix /env/ \ - 'r-gpgr ==2.3.0' \ + 'r-gpgr ==2.3.1' \ 'r-sigrap ==0.1.1' \ 'bioconductor-bsgenome.hsapiens.ucsc.hg38 ==1.4.5' \ 'bioconductor-txdb.hsapiens.ucsc.hg38.knowngene ==3.16.0' \ From d34bc059a6ead7a87017083c6a227490974a3797 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Tue, 2 Jun 2026 10:02:53 +1000 Subject: [PATCH 60/89] fix: skip PCGR when variant count exceeds MAX_SOMATIC_VARIANTS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When select_pcgr_variants cannot bring the PASS count below MAX_SOMATIC_VARIANTS via tiered filtering it raises RuntimeError, aborting the entire report step. This left sash with no usable SMLV_SOMATIC_REPORT output for samples with very high variant counts (e.g. L2100242 with 595,416 PASS variants — high CNA complexity, not a true hypermutator). Catch the RuntimeError, log a warning, and skip prepare_vcf_somatic and run_somatic. Non-PCGR outputs (bcftools stats, AF distributions, variant counts) continue to publish. Companion sash change marks the PCGR emits as optional. Refs: umccr/sash#52 --- bolt/workflows/smlv_somatic/report.py | 55 ++++++++++++++++----------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/bolt/workflows/smlv_somatic/report.py b/bolt/workflows/smlv_somatic/report.py index 66c43c0..be18d10 100644 --- a/bolt/workflows/smlv_somatic/report.py +++ b/bolt/workflows/smlv_somatic/report.py @@ -115,36 +115,45 @@ def entry(ctx, **kwargs): # PCGR report purple_data = parse_purple_purity_file(kwargs['purple_purity_fp']) + pcgr_skipped = False if variant_counts_process['filter_pass'] <= constants.MAX_SOMATIC_VARIANTS: pcgr_input_vcf_fp = kwargs['vcf_fp'] else: - pcgr_input_vcf_fp = select_pcgr_variants( - kwargs['vcf_fp'], - kwargs['cancer_genes_fp'], + try: + pcgr_input_vcf_fp = select_pcgr_variants( + kwargs['vcf_fp'], + kwargs['cancer_genes_fp'], + kwargs['tumor_name'], + output_dir, + ) + except RuntimeError as e: + # NOTE(QC): tiered filtering could not bring PASS count below + # MAX_SOMATIC_VARIANTS (sash #52). Skip PCGR; sash marks the + # PCGR emits as optional so downstream reports still publish. + logger.warning(f'Skipping PCGR for {kwargs["tumor_name"]}: {e}') + pcgr_skipped = True + + if not pcgr_skipped: + pcgr_prep_fp = pcgr.prepare_vcf_somatic( + pcgr_input_vcf_fp, kwargs['tumor_name'], + kwargs['normal_name'], output_dir, ) - pcgr_prep_fp = pcgr.prepare_vcf_somatic( - pcgr_input_vcf_fp, - kwargs['tumor_name'], - kwargs['normal_name'], - output_dir, - ) - - pcgr_output_dir = output_dir / 'pcgr' - pcgr.run_somatic( - pcgr_prep_fp, - kwargs['pcgr_data_dir'], - kwargs['vep_dir'], - pcgr_output_dir, - threads=kwargs['threads'], - pcgr_conda=kwargs['pcgr_conda'], - pcgrr_conda=kwargs['pcgrr_conda'], - purity=purple_data['purity'], - ploidy=purple_data['ploidy'], - sample_id=kwargs['tumor_name'], - ) + pcgr_output_dir = output_dir / 'pcgr' + pcgr.run_somatic( + pcgr_prep_fp, + kwargs['pcgr_data_dir'], + kwargs['vep_dir'], + pcgr_output_dir, + threads=kwargs['threads'], + pcgr_conda=kwargs['pcgr_conda'], + pcgrr_conda=kwargs['pcgrr_conda'], + purity=purple_data['purity'], + ploidy=purple_data['ploidy'], + sample_id=kwargs['tumor_name'], + ) def bcftools_stats_prepare(input_fp, tumor_name, output_dir): From 1c1d023fa50c1d497022126e3e78196bb613ebe5 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Tue, 2 Jun 2026 10:55:28 +1000 Subject: [PATCH 61/89] test: cover PCGR graceful skip on unresolvable variant count overflow Add three tests: - select_pcgr_variants raises RuntimeError when all SAGE_HOTSPOT variants (RETAIN_FIELDS) make tiered filtering impossible - entry() skips prepare_vcf_somatic + run_somatic when the RuntimeError is caught (core of the sash #52 fix) - entry() calls run_somatic normally when count is within the limit (regression guard) --- tests/test_pcgr_hypermutated.py | 108 ++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/tests/test_pcgr_hypermutated.py b/tests/test_pcgr_hypermutated.py index 93e4390..2f7e4f0 100644 --- a/tests/test_pcgr_hypermutated.py +++ b/tests/test_pcgr_hypermutated.py @@ -5,6 +5,8 @@ import unittest from unittest.mock import patch +from click.testing import CliRunner + import cyvcf2 import bolt.common.constants as constants @@ -535,5 +537,111 @@ def test_annotation_filter_excluded_from_annotated_count(self): self.assertEqual(counts['dragen'], 2) +class TestSelectPcgrVariantsRaisesOnUnresolvableOverflow(unittest.TestCase): + """select_pcgr_variants raises RuntimeError when all variants are retained (hotspots).""" + + def _fake_execute(self, vcf_fp): + def _run(cmd, **_): + import re + m = re.search(r'--output\s+(\S+)', cmd) + if m and 'bcftools annotate' in cmd: + shutil.copy(str(vcf_fp), m.group(1)) + else: + util.execute_command(cmd) + return _run + + def test_raises_when_all_variants_are_hotspots(self): + """All SAGE_HOTSPOT variants are RETAIN_FIELDS — tiered filtering cannot drop any; RuntimeError expected.""" + # RETAIN_FIELDS_FILTERING includes SAGE_HOTSPOT — use that flag, not HMF_HOTSPOT + HOTSPOT_INFO = f'SAGE_HOTSPOT;PCGR_ACTIONABILITY_TIER=1;PCGR_CSQ={_csq("intron_variant")}' + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + vcf_fp = tmp_path / 'input.vcf' + variants = [(i * 10, HOTSPOT_INFO) for i in range(1, 6)] + _write_vcf(vcf_fp, variants) + cancer_genes = tmp_path / 'genes.bed' + cancer_genes.write_text('chr1\t1\t9999999\n') + + with patch('bolt.common.constants.MAX_SOMATIC_VARIANTS', 3), \ + patch('bolt.util.execute_command', side_effect=self._fake_execute(vcf_fp)): + with self.assertRaises(RuntimeError): + report_mod.select_pcgr_variants(vcf_fp, cancer_genes, 'TUMOR', tmp_path) + + +_PASS_COUNTS = {'pass': {'snps': 0, 'indels': 0, 'others': 0, 'total': 0}} + + +def _cli_args(dummy, output_dir): + """Return CliRunner args list for report entry(); all file paths point to dummy.""" + d = str(dummy) + return [ + '--tumor_name', 'TUMOR', + '--normal_name', 'NORMAL', + '--vcf_fp', d, + '--vcf_filters_fp', d, + '--vcf_dragen_fp', d, + '--vep_dir', str(dummy.parent), + '--purple_purity_fp', d, + '--cancer_genes_fp', d, + '--giab_regions_fp', d, + '--genome_fp', d, + '--threads', '1', + '--output_dir', str(output_dir), + ] + + +class TestEntrySkipsPcgrOnOverflow(unittest.TestCase): + """entry() catches RuntimeError from select_pcgr_variants and skips PCGR entirely.""" + + def test_run_somatic_not_called_when_cap_exceeded(self): + """When select_pcgr_variants raises RuntimeError, run_somatic must not be called.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + dummy = tmp_path / 'dummy.vcf.gz' + dummy.touch() + + with patch.object(report_mod, 'bcftools_stats_prepare', return_value=dummy), \ + patch.object(report_mod, 'run_bcftools_stats'), \ + patch.object(report_mod, 'allele_frequencies'), \ + patch.object(report_mod, 'count_variant_types', return_value=_PASS_COUNTS), \ + patch.object(report_mod, 'count_variant_process', + return_value={'filter_pass': constants.MAX_SOMATIC_VARIANTS + 1}), \ + patch.object(report_mod, 'parse_purple_purity_file', + return_value={'purity': 0.8, 'ploidy': 2.0}), \ + patch.object(report_mod, 'select_pcgr_variants', + side_effect=RuntimeError('595416 > 450000')), \ + patch.object(pcgr, 'prepare_vcf_somatic') as mock_prep, \ + patch.object(pcgr, 'run_somatic') as mock_run: + result = CliRunner().invoke(report_mod.entry, _cli_args(dummy, tmp_path / 'out')) + + self.assertEqual(result.exit_code, 0, result.output) + mock_prep.assert_not_called() + mock_run.assert_not_called() + + def test_run_somatic_called_when_within_limit(self): + """When PASS count is within limit, run_somatic must be called normally.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + dummy = tmp_path / 'dummy.vcf.gz' + dummy.touch() + fake_prep_output = tmp_path / 'prep.vcf.gz' + fake_prep_output.touch() + + with patch.object(report_mod, 'bcftools_stats_prepare', return_value=dummy), \ + patch.object(report_mod, 'run_bcftools_stats'), \ + patch.object(report_mod, 'allele_frequencies'), \ + patch.object(report_mod, 'count_variant_types', return_value=_PASS_COUNTS), \ + patch.object(report_mod, 'count_variant_process', + return_value={'filter_pass': constants.MAX_SOMATIC_VARIANTS - 1}), \ + patch.object(report_mod, 'parse_purple_purity_file', + return_value={'purity': 0.8, 'ploidy': 2.0}), \ + patch.object(pcgr, 'prepare_vcf_somatic', return_value=fake_prep_output), \ + patch.object(pcgr, 'run_somatic') as mock_run: + result = CliRunner().invoke(report_mod.entry, _cli_args(dummy, tmp_path / 'out')) + + self.assertEqual(result.exit_code, 0, result.output) + mock_run.assert_called_once() + + if __name__ == '__main__': unittest.main() From 162c689a2e959b5c133cb45a09ebd9ace49c9916 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Wed, 3 Jun 2026 11:36:41 +1000 Subject: [PATCH 62/89] fix: treat PCGR_MUTATION_HOTSPOT=. as missing in retention check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PCGR writes PCGR_MUTATION_HOTSPOT=. (Type=String placeholder) on every non-hotspot variant. cyvcf2 returns the string '.' which Python evaluates as truthy, so any(variant.INFO.get(e) ...) always returned True — ALL variants were treated as retained, variants_sorted stayed empty, and select_pcgr_variants raised RuntimeError for any sample with >450k PASS variants (sash #52 root cause). Fix: exclude '.' alongside None so only genuinely set String/Flag fields trigger retention. Adds regression test with PCGR_MUTATION_HOTSPOT=. fixture to prevent silent recurrence. --- bolt/workflows/smlv_somatic/report.py | 2 +- tests/test_pcgr_hypermutated.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/bolt/workflows/smlv_somatic/report.py b/bolt/workflows/smlv_somatic/report.py index be18d10..75968c0 100644 --- a/bolt/workflows/smlv_somatic/report.py +++ b/bolt/workflows/smlv_somatic/report.py @@ -345,7 +345,7 @@ def select_pcgr_variants(vcf_fp, cancer_genes_fp, tumor_name, output_dir): for variant_count, variant in enumerate(cyvcf2.VCF(fp_annotated_out), 1): variant_repr = pcgr.get_variant_repr(variant) - if any(variant.INFO.get(e) for e in constants.RETAIN_FIELDS_FILTERING): + if any(variant.INFO.get(e) not in (None, '.') for e in constants.RETAIN_FIELDS_FILTERING): continue data = pcgr.get_variant_filter_data(variant) diff --git a/tests/test_pcgr_hypermutated.py b/tests/test_pcgr_hypermutated.py index 2f7e4f0..35db782 100644 --- a/tests/test_pcgr_hypermutated.py +++ b/tests/test_pcgr_hypermutated.py @@ -174,6 +174,24 @@ def test_retained_variants_bypass_tiered_filter(self): count = self._run(v, limit=4, tmp=tmp) self.assertEqual(count, 4) + def test_pcgr_mutation_hotspot_dot_not_treated_as_retained(self): + """PCGR_MUTATION_HOTSPOT=. must not retain variants — '.' is a missing-value placeholder. + + cyvcf2 returns the string '.' (truthy) for String INFO fields written as '=.' by PCGR on + every non-hotspot variant. Without the fix, any(variant.INFO.get(e) ...) always returns + True and ALL variants are treated as retained, so tiered filtering never drops anything and + RuntimeError fires for any sample with >450k variants (sash #52 root cause). + """ + with tempfile.TemporaryDirectory() as tmp: + v = [] + for i in range(1, 3): # 2 real SAGE_HOTSPOT — must be retained + v.append((i*10, f'SAGE_HOTSPOT;PCGR_MUTATION_HOTSPOT=.;PCGR_ACTIONABILITY_TIER=1;PCGR_CSQ={_csq("intron_variant")}')) + for i in range(3, 8): # 5 NONCODING with PCGR_MUTATION_HOTSPOT=. — must be droppable + v.append((i*10, f'PCGR_MUTATION_HOTSPOT=.;PCGR_ACTIONABILITY_TIER=N;PCGR_CSQ={_csq("intergenic_variant")}')) + # limit=2: only the 2 real SAGE_HOTSPOT variants survive; the 5 dot-placeholder ones are dropped + count = self._run(v, limit=2, tmp=tmp) + self.assertEqual(count, 2) + def test_filters_set_vcf_marks_dropped_variants(self): """The traceability VCF marks filtered-out variants with PCGR_count_limit. From eb19562b9c34617fc6698f5e748676dd30cdf9e9 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Wed, 3 Jun 2026 11:37:40 +1000 Subject: [PATCH 63/89] refactor: tighten retention-check comment to one line --- bolt/workflows/smlv_somatic/report.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bolt/workflows/smlv_somatic/report.py b/bolt/workflows/smlv_somatic/report.py index 75968c0..2501b54 100644 --- a/bolt/workflows/smlv_somatic/report.py +++ b/bolt/workflows/smlv_somatic/report.py @@ -345,6 +345,7 @@ def select_pcgr_variants(vcf_fp, cancer_genes_fp, tumor_name, output_dir): for variant_count, variant in enumerate(cyvcf2.VCF(fp_annotated_out), 1): variant_repr = pcgr.get_variant_repr(variant) + # NOTE(QC): exclude '.' — PCGR writes it as a missing-value placeholder for String fields; cyvcf2 returns it truthy (sash #52). if any(variant.INFO.get(e) not in (None, '.') for e in constants.RETAIN_FIELDS_FILTERING): continue From 439e9e8372fe680eface8db6810da4251d28ea16 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Wed, 3 Jun 2026 11:44:09 +1000 Subject: [PATCH 64/89] test: assert real PCGR_MUTATION_HOTSPOT value retains variant --- tests/test_pcgr_hypermutated.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_pcgr_hypermutated.py b/tests/test_pcgr_hypermutated.py index 35db782..2082c4a 100644 --- a/tests/test_pcgr_hypermutated.py +++ b/tests/test_pcgr_hypermutated.py @@ -174,6 +174,17 @@ def test_retained_variants_bypass_tiered_filter(self): count = self._run(v, limit=4, tmp=tmp) self.assertEqual(count, 4) + def test_pcgr_mutation_hotspot_real_value_is_retained(self): + """A real PCGR_MUTATION_HOTSPOT value (non-dot) must retain the variant.""" + with tempfile.TemporaryDirectory() as tmp: + v = [] + for i in range(1, 3): # 2 real hotspot variants — must survive + v.append((i*10, f'PCGR_MUTATION_HOTSPOT=GRCH38_1_{i}_A_T;PCGR_ACTIONABILITY_TIER=1;PCGR_CSQ={_csq("intron_variant")}')) + for i in range(3, 8): # 5 NONCODING — dropped + v.append((i*10, f'PCGR_ACTIONABILITY_TIER=N;PCGR_CSQ={_csq("intergenic_variant")}')) + count = self._run(v, limit=2, tmp=tmp) + self.assertEqual(count, 2) + def test_pcgr_mutation_hotspot_dot_not_treated_as_retained(self): """PCGR_MUTATION_HOTSPOT=. must not retain variants — '.' is a missing-value placeholder. From 3c27cb008da20fb5d38b55829b2c74a4d181ac89 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen <37511834+qclayssen@users.noreply.github.com> Date: Tue, 9 Jun 2026 14:59:43 +1000 Subject: [PATCH 65/89] change: drop --estimate_signatures from PCGR somatic invocation (#31) * fix: drop --estimate_signatures from PCGR somatic invocation PCGR's mutational signature estimation is unreliable for our cohort and we get signature analysis from gpgr/sigrap downstream. Keep --estimate_msi and --estimate_tmb (still useful in the PCGR report). Refs: umccr/sash#57 * Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- bolt/common/pcgr.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bolt/common/pcgr.py b/bolt/common/pcgr.py index fc2151c..b94ae47 100644 --- a/bolt/common/pcgr.py +++ b/bolt/common/pcgr.py @@ -147,7 +147,8 @@ def run_somatic(input_fp, pcgr_refdata_dir, vep_dir, output_dir, chunk_nbr=None, f'--control_af_tag NORMAL_AF', f'--genome_assembly grch38', f'--assay WGS', - *([] if disable_estimates else ['--estimate_signatures', '--estimate_msi', '--estimate_tmb']), + # NOTE: --estimate_signatures dropped (umccr/sash#57); keep msi/tmb. + *([] if disable_estimates else ['--estimate_msi', '--estimate_tmb']), f'--vcfanno_n_proc {vcfanno_threads}', f'--vep_n_forks {vep_forks}', f'--vep_pick_order biotype,rank,appris,tsl,ccds,canonical,length,mane_plus_clinical,mane_select', From 819fda88b20c2765ce570d67988a45910300ad5b Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 3 Jul 2026 15:53:26 +1000 Subject: [PATCH 66/89] docs: update CHANGELOG for fixes landed since v0.3.1 tag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Documents CI smoke tests, build fixes, split_vcf tabix-indexing, r-gpgr 2.3.1 bump, and the estimate_signatures drop (#31, sash#57) — all merged into release/0.3.0 after the v0.3.1 tag but never recorded. --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9eadff9..8df7a78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,12 @@ - Fix `split_vcf` writing uncompressed plain `.vcf` chunks — now uses `.vcf.gz` with `wz` mode - Fix `PCGR_ACTIONABILITY_TIER` VCF header description — updated to match stored short-form values (`1`,`2`,`3`,`4`,`N`) - Add regression test for chunk file compression (`test_chunks_are_gzipped`) +- Add CI smoke tests to catch Docker image startup failures before push +- Fix `build.yaml` and `Dockerfile.pcgr` build issues +- Remove unused `logging` import and fix `PCGR_MAX_SOMATIC_VARIANTS` header description in `constants.py` +- Fix `split_vcf` chunks not tabix-indexed, causing PCGR to fail reading them; add regression test +- Bump `r-gpgr` to 2.3.1 in `Dockerfile.gpgr` +- [31](https://github.com/umccr/bolt/pull/31) - Drop `--estimate_signatures` from PCGR somatic invocation — signature analysis comes from gpgr/sigrap downstream; keep `--estimate_msi`/`--estimate_tmb` ([sash#57](https://github.com/umccr/sash/issues/57)) ## 0.3.0 From 97446f7ad24d401b0477249bb5894c6fb2b58f69 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen <37511834+qclayssen@users.noreply.github.com> Date: Fri, 3 Jul 2026 15:54:14 +1000 Subject: [PATCH 67/89] Fix/52 pcgr skip high variant count (#32) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: skip PCGR when variant count exceeds MAX_SOMATIC_VARIANTS When select_pcgr_variants cannot bring the PASS count below MAX_SOMATIC_VARIANTS via tiered filtering it raises RuntimeError, aborting the entire report step. This left sash with no usable SMLV_SOMATIC_REPORT output for samples with very high variant counts (e.g. L2100242 with 595,416 PASS variants — high CNA complexity, not a true hypermutator). Catch the RuntimeError, log a warning, and skip prepare_vcf_somatic and run_somatic. Non-PCGR outputs (bcftools stats, AF distributions, variant counts) continue to publish. Companion sash change marks the PCGR emits as optional. Refs: umccr/sash#52 * test: cover PCGR graceful skip on unresolvable variant count overflow Add three tests: - select_pcgr_variants raises RuntimeError when all SAGE_HOTSPOT variants (RETAIN_FIELDS) make tiered filtering impossible - entry() skips prepare_vcf_somatic + run_somatic when the RuntimeError is caught (core of the sash #52 fix) - entry() calls run_somatic normally when count is within the limit (regression guard) * fix: treat PCGR_MUTATION_HOTSPOT=. as missing in retention check PCGR writes PCGR_MUTATION_HOTSPOT=. (Type=String placeholder) on every non-hotspot variant. cyvcf2 returns the string '.' which Python evaluates as truthy, so any(variant.INFO.get(e) ...) always returned True — ALL variants were treated as retained, variants_sorted stayed empty, and select_pcgr_variants raised RuntimeError for any sample with >450k PASS variants (sash #52 root cause). Fix: exclude '.' alongside None so only genuinely set String/Flag fields trigger retention. Adds regression test with PCGR_MUTATION_HOTSPOT=. fixture to prevent silent recurrence. * refactor: tighten retention-check comment to one line * test: assert real PCGR_MUTATION_HOTSPOT value retains variant --- bolt/workflows/smlv_somatic/report.py | 58 ++++++----- tests/test_pcgr_hypermutated.py | 137 ++++++++++++++++++++++++++ 2 files changed, 171 insertions(+), 24 deletions(-) diff --git a/bolt/workflows/smlv_somatic/report.py b/bolt/workflows/smlv_somatic/report.py index 66c43c0..2501b54 100644 --- a/bolt/workflows/smlv_somatic/report.py +++ b/bolt/workflows/smlv_somatic/report.py @@ -115,36 +115,45 @@ def entry(ctx, **kwargs): # PCGR report purple_data = parse_purple_purity_file(kwargs['purple_purity_fp']) + pcgr_skipped = False if variant_counts_process['filter_pass'] <= constants.MAX_SOMATIC_VARIANTS: pcgr_input_vcf_fp = kwargs['vcf_fp'] else: - pcgr_input_vcf_fp = select_pcgr_variants( - kwargs['vcf_fp'], - kwargs['cancer_genes_fp'], + try: + pcgr_input_vcf_fp = select_pcgr_variants( + kwargs['vcf_fp'], + kwargs['cancer_genes_fp'], + kwargs['tumor_name'], + output_dir, + ) + except RuntimeError as e: + # NOTE(QC): tiered filtering could not bring PASS count below + # MAX_SOMATIC_VARIANTS (sash #52). Skip PCGR; sash marks the + # PCGR emits as optional so downstream reports still publish. + logger.warning(f'Skipping PCGR for {kwargs["tumor_name"]}: {e}') + pcgr_skipped = True + + if not pcgr_skipped: + pcgr_prep_fp = pcgr.prepare_vcf_somatic( + pcgr_input_vcf_fp, kwargs['tumor_name'], + kwargs['normal_name'], output_dir, ) - pcgr_prep_fp = pcgr.prepare_vcf_somatic( - pcgr_input_vcf_fp, - kwargs['tumor_name'], - kwargs['normal_name'], - output_dir, - ) - - pcgr_output_dir = output_dir / 'pcgr' - pcgr.run_somatic( - pcgr_prep_fp, - kwargs['pcgr_data_dir'], - kwargs['vep_dir'], - pcgr_output_dir, - threads=kwargs['threads'], - pcgr_conda=kwargs['pcgr_conda'], - pcgrr_conda=kwargs['pcgrr_conda'], - purity=purple_data['purity'], - ploidy=purple_data['ploidy'], - sample_id=kwargs['tumor_name'], - ) + pcgr_output_dir = output_dir / 'pcgr' + pcgr.run_somatic( + pcgr_prep_fp, + kwargs['pcgr_data_dir'], + kwargs['vep_dir'], + pcgr_output_dir, + threads=kwargs['threads'], + pcgr_conda=kwargs['pcgr_conda'], + pcgrr_conda=kwargs['pcgrr_conda'], + purity=purple_data['purity'], + ploidy=purple_data['ploidy'], + sample_id=kwargs['tumor_name'], + ) def bcftools_stats_prepare(input_fp, tumor_name, output_dir): @@ -336,7 +345,8 @@ def select_pcgr_variants(vcf_fp, cancer_genes_fp, tumor_name, output_dir): for variant_count, variant in enumerate(cyvcf2.VCF(fp_annotated_out), 1): variant_repr = pcgr.get_variant_repr(variant) - if any(variant.INFO.get(e) for e in constants.RETAIN_FIELDS_FILTERING): + # NOTE(QC): exclude '.' — PCGR writes it as a missing-value placeholder for String fields; cyvcf2 returns it truthy (sash #52). + if any(variant.INFO.get(e) not in (None, '.') for e in constants.RETAIN_FIELDS_FILTERING): continue data = pcgr.get_variant_filter_data(variant) diff --git a/tests/test_pcgr_hypermutated.py b/tests/test_pcgr_hypermutated.py index 93e4390..2082c4a 100644 --- a/tests/test_pcgr_hypermutated.py +++ b/tests/test_pcgr_hypermutated.py @@ -5,6 +5,8 @@ import unittest from unittest.mock import patch +from click.testing import CliRunner + import cyvcf2 import bolt.common.constants as constants @@ -172,6 +174,35 @@ def test_retained_variants_bypass_tiered_filter(self): count = self._run(v, limit=4, tmp=tmp) self.assertEqual(count, 4) + def test_pcgr_mutation_hotspot_real_value_is_retained(self): + """A real PCGR_MUTATION_HOTSPOT value (non-dot) must retain the variant.""" + with tempfile.TemporaryDirectory() as tmp: + v = [] + for i in range(1, 3): # 2 real hotspot variants — must survive + v.append((i*10, f'PCGR_MUTATION_HOTSPOT=GRCH38_1_{i}_A_T;PCGR_ACTIONABILITY_TIER=1;PCGR_CSQ={_csq("intron_variant")}')) + for i in range(3, 8): # 5 NONCODING — dropped + v.append((i*10, f'PCGR_ACTIONABILITY_TIER=N;PCGR_CSQ={_csq("intergenic_variant")}')) + count = self._run(v, limit=2, tmp=tmp) + self.assertEqual(count, 2) + + def test_pcgr_mutation_hotspot_dot_not_treated_as_retained(self): + """PCGR_MUTATION_HOTSPOT=. must not retain variants — '.' is a missing-value placeholder. + + cyvcf2 returns the string '.' (truthy) for String INFO fields written as '=.' by PCGR on + every non-hotspot variant. Without the fix, any(variant.INFO.get(e) ...) always returns + True and ALL variants are treated as retained, so tiered filtering never drops anything and + RuntimeError fires for any sample with >450k variants (sash #52 root cause). + """ + with tempfile.TemporaryDirectory() as tmp: + v = [] + for i in range(1, 3): # 2 real SAGE_HOTSPOT — must be retained + v.append((i*10, f'SAGE_HOTSPOT;PCGR_MUTATION_HOTSPOT=.;PCGR_ACTIONABILITY_TIER=1;PCGR_CSQ={_csq("intron_variant")}')) + for i in range(3, 8): # 5 NONCODING with PCGR_MUTATION_HOTSPOT=. — must be droppable + v.append((i*10, f'PCGR_MUTATION_HOTSPOT=.;PCGR_ACTIONABILITY_TIER=N;PCGR_CSQ={_csq("intergenic_variant")}')) + # limit=2: only the 2 real SAGE_HOTSPOT variants survive; the 5 dot-placeholder ones are dropped + count = self._run(v, limit=2, tmp=tmp) + self.assertEqual(count, 2) + def test_filters_set_vcf_marks_dropped_variants(self): """The traceability VCF marks filtered-out variants with PCGR_count_limit. @@ -535,5 +566,111 @@ def test_annotation_filter_excluded_from_annotated_count(self): self.assertEqual(counts['dragen'], 2) +class TestSelectPcgrVariantsRaisesOnUnresolvableOverflow(unittest.TestCase): + """select_pcgr_variants raises RuntimeError when all variants are retained (hotspots).""" + + def _fake_execute(self, vcf_fp): + def _run(cmd, **_): + import re + m = re.search(r'--output\s+(\S+)', cmd) + if m and 'bcftools annotate' in cmd: + shutil.copy(str(vcf_fp), m.group(1)) + else: + util.execute_command(cmd) + return _run + + def test_raises_when_all_variants_are_hotspots(self): + """All SAGE_HOTSPOT variants are RETAIN_FIELDS — tiered filtering cannot drop any; RuntimeError expected.""" + # RETAIN_FIELDS_FILTERING includes SAGE_HOTSPOT — use that flag, not HMF_HOTSPOT + HOTSPOT_INFO = f'SAGE_HOTSPOT;PCGR_ACTIONABILITY_TIER=1;PCGR_CSQ={_csq("intron_variant")}' + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + vcf_fp = tmp_path / 'input.vcf' + variants = [(i * 10, HOTSPOT_INFO) for i in range(1, 6)] + _write_vcf(vcf_fp, variants) + cancer_genes = tmp_path / 'genes.bed' + cancer_genes.write_text('chr1\t1\t9999999\n') + + with patch('bolt.common.constants.MAX_SOMATIC_VARIANTS', 3), \ + patch('bolt.util.execute_command', side_effect=self._fake_execute(vcf_fp)): + with self.assertRaises(RuntimeError): + report_mod.select_pcgr_variants(vcf_fp, cancer_genes, 'TUMOR', tmp_path) + + +_PASS_COUNTS = {'pass': {'snps': 0, 'indels': 0, 'others': 0, 'total': 0}} + + +def _cli_args(dummy, output_dir): + """Return CliRunner args list for report entry(); all file paths point to dummy.""" + d = str(dummy) + return [ + '--tumor_name', 'TUMOR', + '--normal_name', 'NORMAL', + '--vcf_fp', d, + '--vcf_filters_fp', d, + '--vcf_dragen_fp', d, + '--vep_dir', str(dummy.parent), + '--purple_purity_fp', d, + '--cancer_genes_fp', d, + '--giab_regions_fp', d, + '--genome_fp', d, + '--threads', '1', + '--output_dir', str(output_dir), + ] + + +class TestEntrySkipsPcgrOnOverflow(unittest.TestCase): + """entry() catches RuntimeError from select_pcgr_variants and skips PCGR entirely.""" + + def test_run_somatic_not_called_when_cap_exceeded(self): + """When select_pcgr_variants raises RuntimeError, run_somatic must not be called.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + dummy = tmp_path / 'dummy.vcf.gz' + dummy.touch() + + with patch.object(report_mod, 'bcftools_stats_prepare', return_value=dummy), \ + patch.object(report_mod, 'run_bcftools_stats'), \ + patch.object(report_mod, 'allele_frequencies'), \ + patch.object(report_mod, 'count_variant_types', return_value=_PASS_COUNTS), \ + patch.object(report_mod, 'count_variant_process', + return_value={'filter_pass': constants.MAX_SOMATIC_VARIANTS + 1}), \ + patch.object(report_mod, 'parse_purple_purity_file', + return_value={'purity': 0.8, 'ploidy': 2.0}), \ + patch.object(report_mod, 'select_pcgr_variants', + side_effect=RuntimeError('595416 > 450000')), \ + patch.object(pcgr, 'prepare_vcf_somatic') as mock_prep, \ + patch.object(pcgr, 'run_somatic') as mock_run: + result = CliRunner().invoke(report_mod.entry, _cli_args(dummy, tmp_path / 'out')) + + self.assertEqual(result.exit_code, 0, result.output) + mock_prep.assert_not_called() + mock_run.assert_not_called() + + def test_run_somatic_called_when_within_limit(self): + """When PASS count is within limit, run_somatic must be called normally.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + dummy = tmp_path / 'dummy.vcf.gz' + dummy.touch() + fake_prep_output = tmp_path / 'prep.vcf.gz' + fake_prep_output.touch() + + with patch.object(report_mod, 'bcftools_stats_prepare', return_value=dummy), \ + patch.object(report_mod, 'run_bcftools_stats'), \ + patch.object(report_mod, 'allele_frequencies'), \ + patch.object(report_mod, 'count_variant_types', return_value=_PASS_COUNTS), \ + patch.object(report_mod, 'count_variant_process', + return_value={'filter_pass': constants.MAX_SOMATIC_VARIANTS - 1}), \ + patch.object(report_mod, 'parse_purple_purity_file', + return_value={'purity': 0.8, 'ploidy': 2.0}), \ + patch.object(pcgr, 'prepare_vcf_somatic', return_value=fake_prep_output), \ + patch.object(pcgr, 'run_somatic') as mock_run: + result = CliRunner().invoke(report_mod.entry, _cli_args(dummy, tmp_path / 'out')) + + self.assertEqual(result.exit_code, 0, result.output) + mock_run.assert_called_once() + + if __name__ == '__main__': unittest.main() From 76b71ac058d1c4d3a87607a7cec8f73d90093de0 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 3 Jul 2026 16:04:14 +1000 Subject: [PATCH 68/89] fix: pin free-disk-space action, disable_estimates in chunked PCGR, logger.info cleanup --- .github/workflows/build.yaml | 2 +- bolt/common/pcgr.py | 26 +++++-------------------- bolt/workflows/smlv_somatic/annotate.py | 2 +- 3 files changed, 7 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 22018a8..9fd84a5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -18,7 +18,7 @@ jobs: uses: actions/checkout@v4 - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main + uses: jlumbroso/free-disk-space@v1.3.0 with: # this might remove tools that are actually needed, # if set to "true" but frees about 6 GB diff --git a/bolt/common/pcgr.py b/bolt/common/pcgr.py index b94ae47..5609d13 100644 --- a/bolt/common/pcgr.py +++ b/bolt/common/pcgr.py @@ -15,7 +15,6 @@ from .. import util from ..common import constants -# Use the existing logger configuration logger = logging.getLogger(__name__) def prepare_vcf_somatic(input_fp, tumor_name, normal_name, output_dir): @@ -130,7 +129,6 @@ def run_somatic(input_fp, pcgr_refdata_dir, vep_dir, output_dir, chunk_nbr=None, logger.warning(f"Output directory '{output_dir}' already exists and will be overwritten") shutil.rmtree(output_dir) - # Create output directory output_dir.mkdir(parents=True, exist_ok=True) if not sample_id: @@ -229,7 +227,6 @@ def run_germline(input_fp, panel_fp, pcgr_refdata_dir, vep_dir, output_dir, thre logger.warning(f"Output directory '{cpsr_output_dir}' already exists and will be overwritten") shutil.rmtree(cpsr_output_dir) - # Create output directory cpsr_output_dir.mkdir(parents=True, exist_ok=True) command_args = [ @@ -416,17 +413,11 @@ def collect_cpsr_annotation_data(tsv_fp, vcf_fp, info_field_map): return compile_annotation_data(data_tsv, data_vcf) def parse_genomic_change(genomic_change): - """ - Parse a genomic change string, e.g., "3:g.41224645T>C" - Returns a tuple: (chrom, pos, ref, alt) - """ - # Regular expression for the format "chrom:g.posRef>Alt" + # Format: "chrom:g.posRef>Alt" e.g. "3:g.41224645T>C" pattern = r'^(?P\w+):g\.(?P\d+)(?P\w+)>(?P\w+)$' match = re.match(pattern, genomic_change) if not match: raise ValueError(f"Format not recognized: {genomic_change}") - - # Get values and format as needed chrom = f"chr{match.group('chrom')}" pos = int(match.group('pos')) ref = match.group('ref') @@ -516,11 +507,6 @@ def annotate_record(record, annotations, *, allow_missing=False): return record def split_vcf(input_vcf, output_dir, *, max_variants=None): - """ - Splits a VCF file into multiple chunks, each containing up to max_variants variants. - Each chunk includes the VCF header. - Ensures no overlapping positions between chunks. - """ if max_variants is None: max_variants = constants.MAX_SOMATIC_VARIANTS elif max_variants <= 0: @@ -568,27 +554,25 @@ def split_vcf(input_vcf, output_dir, *, max_variants=None): def run_somatic_chunk(vcf_chunks, pcgr_data_dir, vep_dir, output_dir, pcgr_output_dir, max_threads, pcgr_conda, pcgrr_conda): pcgr_tsv_files = [] pcgr_vcf_files = [] - - # Process each chunk sequentially + for chunk_number, vcf_file in enumerate(vcf_chunks, start=1): - pcgr_tsv_fp, pcgr_vcf_fp = run_somatic(vcf_file, pcgr_data_dir, vep_dir, pcgr_output_dir, chunk_nbr=chunk_number, threads=max_threads, pcgr_conda=pcgr_conda, pcgrr_conda=pcgrr_conda) + pcgr_tsv_fp, pcgr_vcf_fp = run_somatic(vcf_file, pcgr_data_dir, vep_dir, pcgr_output_dir, chunk_nbr=chunk_number, threads=max_threads, pcgr_conda=pcgr_conda, pcgrr_conda=pcgrr_conda, disable_estimates=True) if pcgr_tsv_fp: pcgr_tsv_files.append(pcgr_tsv_fp) if pcgr_vcf_fp: pcgr_vcf_files.append(pcgr_vcf_fp) - + merged_vcf_fp, merged_tsv_fp = merging_pcgr_files(output_dir, pcgr_vcf_files, pcgr_tsv_files) return merged_tsv_fp, merged_vcf_fp + def merging_pcgr_files(output_dir, pcgr_vcf_files, pcgr_tsv_files): pcgr_dir = pathlib.Path(output_dir) / 'pcgr' pcgr_dir.mkdir(exist_ok=True) - # Merge all TSV files into a single file in the pcgr directory merged_tsv_fp = pcgr_dir / "nosampleset.pcgr_acmg.grch38.snvs_indels.tiers.tsv.gz" util.merge_tsv_files(pcgr_tsv_files, merged_tsv_fp) - # Step 5: Merge all VCF files into a single file in the pcgr directory merged_vcf_path = pcgr_dir / "nosampleset.pcgr.grch38.pass" merged_vcf = util.merge_vcf_files(pcgr_vcf_files, merged_vcf_path) diff --git a/bolt/workflows/smlv_somatic/annotate.py b/bolt/workflows/smlv_somatic/annotate.py index 01d28b2..1e4e434 100644 --- a/bolt/workflows/smlv_somatic/annotate.py +++ b/bolt/workflows/smlv_somatic/annotate.py @@ -97,7 +97,7 @@ def entry(ctx, **kwargs): pcgr_output_dir = output_dir / 'pcgr' total_variants = util.count_vcf_records(pcgr_prep_fp) - print(f"Total number of variants in the input VCF: {total_variants}") + logger.info(f"Total number of variants in the input VCF: {total_variants}") # Run PCGR in chunks if exceeding the maximum allowed for somatic variants chunk_size = kwargs.get('pcgr_variant_chunk_size') From 0a3a8251dfa937f0ff6e88c67f62d39e6c4c06a6 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 3 Jul 2026 16:05:10 +1000 Subject: [PATCH 69/89] =?UTF-8?q?Bump=20version:=200.3.1=20=E2=86=92=200.3?= =?UTF-8?q?.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- CHANGELOG.md | 11 +++++++++++ pyproject.toml | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index d86a437..8e74266 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.1 +current_version = 0.3.2 commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P[a-z0-9+]+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8df7a78..3d745f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # bolt changelog +## 0.3.2 + +- [32](https://github.com/umccr/bolt/pull/32) - Graceful PCGR skip when `select_pcgr_variants` cannot cap variants to `MAX_SOMATIC_VARIANTS` — pipeline logs a warning and continues without the cancer report rather than hard-failing ([sash#52](https://github.com/umccr/sash/issues/52)) +- Fix `PCGR_MUTATION_HOTSPOT=.` (dot placeholder) being treated as a retained hotspot variant during tiered filtering — was preventing tier-based filtering from running on any sample with >450k PASS variants +- Disable `--estimate_msi`/`--estimate_tmb` in chunked PCGR annotation runs (`split_vcf`) — estimates are not meaningful per-chunk +- Pin `jlumbroso/free-disk-space` CI action to `v1.3.0` (was `@main`) + ## 0.3.1 - Fix `ModuleNotFoundError: No module named 'pkg_resources'` in `bolt:0.3.0-multiqc` — add `setuptools <81` to conda env @@ -14,6 +21,10 @@ - Fix `split_vcf` chunks not tabix-indexed, causing PCGR to fail reading them; add regression test - Bump `r-gpgr` to 2.3.1 in `Dockerfile.gpgr` - [31](https://github.com/umccr/bolt/pull/31) - Drop `--estimate_signatures` from PCGR somatic invocation — signature analysis comes from gpgr/sigrap downstream; keep `--estimate_msi`/`--estimate_tmb` ([sash#57](https://github.com/umccr/sash/issues/57)) +- [32](https://github.com/umccr/bolt/pull/32) - Fix `PCGR_MUTATION_HOTSPOT=.` treated as truthy in retention check, preventing tiered filtering from running for all hypermutated samples ([sash#52](https://github.com/umccr/sash/issues/52)) +- [32](https://github.com/umccr/bolt/pull/32) - Fix graceful PCGR skip when tiered filtering cannot bring PASS count below `MAX_SOMATIC_VARIANTS` — log warning and continue; non-PCGR outputs still publish ([sash#52](https://github.com/umccr/sash/issues/52)) +- Fix: disable MSI/TMB estimates when running PCGR on annotation chunks — estimates on partial VCFs are meaningless +- Fix: pin `jlumbroso/free-disk-space` CI action to `v1.3.0` instead of `@main` ## 0.3.0 diff --git a/pyproject.toml b/pyproject.toml index 6dd2fba..65c0a2f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ include = ["bolt*"] [project] name = "bolt" -version = "0.3.1" +version = "0.3.2" authors = [ {name = "Stephen Watts", email = "stephen.watts@umccr.org"}, ] From 91c5d3c17f4f31b8ee1f2ec2ed824211542e8aed Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 3 Jul 2026 16:05:35 +1000 Subject: [PATCH 70/89] style: remove obvious comments and docstrings from util.py Drops comments that restate what the code already says (logger setup, execute_command flag explanation, merge step annotations) and replaces multi-paragraph docstrings on merge_tsv_files/merge_vcf_files with no docstring, per project conventions. --- bolt/util.py | 62 +++++----------------------------------------------- 1 file changed, 6 insertions(+), 56 deletions(-) diff --git a/bolt/util.py b/bolt/util.py index c3b07fe..3c0af08 100644 --- a/bolt/util.py +++ b/bolt/util.py @@ -11,7 +11,6 @@ from .common import constants -# Set up logging logger = logging.getLogger(__name__) # TODO(SW): create note that number this assumes location of `//` @@ -23,7 +22,6 @@ def get_project_root(): def execute_command(command, log_file_path=None): - # set -e: exit on error, -u: exit on unset variable, -o pipefail: pipeline fails if any command fails prepared_command = f'set -euo pipefail; {textwrap.dedent(command)}' logger.info("Executing command: %s", command.strip()) @@ -129,15 +127,10 @@ def get_qualified_vcf_annotation(anno_enum): return f'{anno_enum.namespace}/{anno_enum.value}' def merge_tsv_files(tsv_files, merged_tsv_fp): - """ - Merge gzipped TSV files into a single gzipped TSV. - """ - with gzip.open(merged_tsv_fp, 'wt', encoding='utf-8') as merged_tsv: for i, tsv_file in enumerate(tsv_files): with gzip.open(tsv_file, 'rt', encoding='utf-8') as infile: for line_number, line in enumerate(infile): - # Skip header except for the first file if i > 0 and line_number == 0: continue merged_tsv.write(line) @@ -145,74 +138,31 @@ def merge_tsv_files(tsv_files, merged_tsv_fp): def merge_vcf_files(vcf_files, merged_vcf_fp): - """ - Merges multiple VCF files into a single sorted VCF file using bcftools. - - Parameters: - - vcf_files: List of paths to VCF files to be merged. - - merged_vcf_fp: Path to the output merged VCF file (without extension). - - Returns: - - Path to the sorted merged VCF file. - """ merged_vcf_fp = pathlib.Path(merged_vcf_fp) merged_unsorted_vcf = merged_vcf_fp.parent / f'{merged_vcf_fp.name}.unsorted.vcf.gz' merged_vcf = merged_vcf_fp.parent / f'{merged_vcf_fp.name}.vcf.gz' - # Prepare the bcftools merge command arguments + delimiter_padding = ' ' * 10 + delimiter = f' \\\n{delimiter_padding}' + command_args = [ 'bcftools merge', '-m all', '-Oz', f'-o {merged_unsorted_vcf}', ] + [str(vcf_file) for vcf_file in vcf_files] + execute_command(f'\n {delimiter.join(command_args)}\n ') - # Format the command for readability - delimiter_padding = ' ' * 10 - delimiter = f' \\\n{delimiter_padding}' - command_args_str = delimiter.join(command_args) - - command = f''' - {command_args_str} - ''' - - # Run the bcftools merge command - logger.info("Running bcftools merge...") - execute_command(command) - logger.info(f"Merged VCF written to: {merged_unsorted_vcf}") - - # Sort the merged VCF file sort_command_args = [ 'bcftools sort', '-Oz', f'-o {merged_vcf}', f'{merged_unsorted_vcf}' ] - sort_command_args_str = delimiter.join(sort_command_args) - sort_command = f''' - {sort_command_args_str} - ''' - - logger.info("Sorting merged VCF file...") - execute_command(sort_command) - logger.info(f"Sorted merged VCF written to: {merged_vcf}") - - # Index the sorted merged VCF file - index_command_args = [ - 'bcftools index', - '-t', - f'{merged_vcf}' - ] - index_command_args_str = delimiter.join(index_command_args) - index_command = f''' - {index_command_args_str} - ''' + execute_command(f'\n {delimiter.join(sort_command_args)}\n ') - logger.info("Indexing sorted merged VCF file...") - execute_command(index_command) - logger.info(f"Indexed merged VCF file: {merged_vcf}.tbi") + execute_command(f'bcftools index -t {merged_vcf}') - # Optionally, remove the unsorted merged VCF file if merged_unsorted_vcf.exists(): merged_unsorted_vcf.unlink() From 70106063cb367d5e1b4ac985835825edf5cc5e4c Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Fri, 3 Jul 2026 16:20:42 +1000 Subject: [PATCH 71/89] docs: clean up CHANGELOG and document test coverage per 0.3.1/0.3.2 - Remove duplicate PR #32 entries that appeared in both 0.3.1 and 0.3.2 - Add test documentation for all new test classes introduced in 0.3.1/0.3.2 - Restore logger.info calls in merge_vcf_files that were removed during cleanup --- CHANGELOG.md | 32 +++++++++++++++++--------------- bolt/util.py | 10 ++++++++-- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d745f2..ddf05e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,13 @@ ## 0.3.2 -- [32](https://github.com/umccr/bolt/pull/32) - Graceful PCGR skip when `select_pcgr_variants` cannot cap variants to `MAX_SOMATIC_VARIANTS` — pipeline logs a warning and continues without the cancer report rather than hard-failing ([sash#52](https://github.com/umccr/sash/issues/52)) -- Fix `PCGR_MUTATION_HOTSPOT=.` (dot placeholder) being treated as a retained hotspot variant during tiered filtering — was preventing tier-based filtering from running on any sample with >450k PASS variants -- Disable `--estimate_msi`/`--estimate_tmb` in chunked PCGR annotation runs (`split_vcf`) — estimates are not meaningful per-chunk -- Pin `jlumbroso/free-disk-space` CI action to `v1.3.0` (was `@main`) +- [32](https://github.com/umccr/bolt/pull/32) - Fix `PCGR_MUTATION_HOTSPOT=.` (dot placeholder) treated as truthy in retention check — was preventing tiered filtering from running for any sample with >450k PASS variants ([sash#52](https://github.com/umccr/sash/issues/52)) +- [32](https://github.com/umccr/bolt/pull/32) - Graceful PCGR skip when `select_pcgr_variants` cannot cap variants to `MAX_SOMATIC_VARIANTS` — logs warning and continues without cancer report; non-PCGR outputs still publish ([sash#52](https://github.com/umccr/sash/issues/52)) +- Fix: disable `--estimate_msi`/`--estimate_tmb` in chunked PCGR annotation runs — estimates on partial VCFs are not meaningful +- Fix: pin `jlumbroso/free-disk-space` CI action to `v1.3.0` (was `@main`) +- Test: `TestSelectPcgrVariants` — 8 integration tests covering tiered trimming, hotspot retention, PANEL retention, NONCODING-first drop order, and the `PCGR_MUTATION_HOTSPOT=.` regression +- Test: `TestEntrySkipsPcgrOnOverflow` — 2 tests: entry() skips PCGR on `RuntimeError` from unresolvable overflow; entry() calls PCGR normally when within limit +- Test: `TestSelectPcgrVariantsRaisesOnUnresolvableOverflow` — asserts `RuntimeError` when retained variants alone exceed `MAX_SOMATIC_VARIANTS` ## 0.3.1 @@ -14,26 +17,25 @@ - Fix VCF writers not closed in `transfer_annotations_somatic` and `transfer_annotations_germline` — BGZip output could be truncated - Fix `split_vcf` writing uncompressed plain `.vcf` chunks — now uses `.vcf.gz` with `wz` mode - Fix `PCGR_ACTIONABILITY_TIER` VCF header description — updated to match stored short-form values (`1`,`2`,`3`,`4`,`N`) -- Add regression test for chunk file compression (`test_chunks_are_gzipped`) -- Add CI smoke tests to catch Docker image startup failures before push +- Fix `split_vcf` chunks not tabix-indexed, causing PCGR to fail reading them - Fix `build.yaml` and `Dockerfile.pcgr` build issues - Remove unused `logging` import and fix `PCGR_MAX_SOMATIC_VARIANTS` header description in `constants.py` -- Fix `split_vcf` chunks not tabix-indexed, causing PCGR to fail reading them; add regression test - Bump `r-gpgr` to 2.3.1 in `Dockerfile.gpgr` +- Add CI smoke tests to catch Docker image startup failures before push - [31](https://github.com/umccr/bolt/pull/31) - Drop `--estimate_signatures` from PCGR somatic invocation — signature analysis comes from gpgr/sigrap downstream; keep `--estimate_msi`/`--estimate_tmb` ([sash#57](https://github.com/umccr/sash/issues/57)) -- [32](https://github.com/umccr/bolt/pull/32) - Fix `PCGR_MUTATION_HOTSPOT=.` treated as truthy in retention check, preventing tiered filtering from running for all hypermutated samples ([sash#52](https://github.com/umccr/sash/issues/52)) -- [32](https://github.com/umccr/bolt/pull/32) - Fix graceful PCGR skip when tiered filtering cannot bring PASS count below `MAX_SOMATIC_VARIANTS` — log warning and continue; non-PCGR outputs still publish ([sash#52](https://github.com/umccr/sash/issues/52)) -- Fix: disable MSI/TMB estimates when running PCGR on annotation chunks — estimates on partial VCFs are meaningless -- Fix: pin `jlumbroso/free-disk-space` CI action to `v1.3.0` instead of `@main` +- Test: regression test for chunk file compression (`test_chunks_are_gzipped`) +- Test: regression test for chunk tabix indexing (`test_chunks_are_tabix_indexed`) +- Test: `TestTierOrdering` — 3 tests verifying `PCGR_TIERS_FILTERING` uses short forms (`N`,`4`,`3`,`2`,`1`) and NONCODING precedes TIER_1 in `get_ordering()` +- Test: `TestSplitVcf` — 4 tests: chunking above/below limit, `.vcf.gz` compression, `.tbi` indexing +- Test: `TestSelectPcgrVariants` — initial 6 integration tests for tiered trimming logic +- Test: `TestGetVariantFilterData`, `TestDetermineFilter`, `TestGetImpacts` — 14 unit tests covering variant attribute extraction and filter-category determination +- Test: `TestCountVariantProcess` — 4 tests: `is_hypermutated` flag, DRAGEN count, SAGE_NOVEL exclusion, annotation-filter exclusion +- Test: `TestRunSomaticChunkArgMapping` — asserts `pcgr_conda` is not shifted into `pcgr_threads` position on positional arg mapping ## 0.3.0 - [17](https://github.com/umccr/bolt/pull/17) - change dragen HRD file optional - - [14](https://github.com/umccr/bolt/pull/14) - gpgr version bump to 2.2.0 - - [3](https://github.com/scwatts/bolt/pull/3) - Improve PCGR / CPSR argument handling - - [6](https://github.com/umccr/bolt/pull/6) - Change oncoanalyser v2.0.0 update, with switch sv caller from GRIPSS to eSVee - - [9](https://github.com/umccr/bolt/pull/9) Add hypermutation sample handling diff --git a/bolt/util.py b/bolt/util.py index 3c0af08..f837314 100644 --- a/bolt/util.py +++ b/bolt/util.py @@ -151,7 +151,10 @@ def merge_vcf_files(vcf_files, merged_vcf_fp): '-Oz', f'-o {merged_unsorted_vcf}', ] + [str(vcf_file) for vcf_file in vcf_files] - execute_command(f'\n {delimiter.join(command_args)}\n ') + command_args_str = delimiter.join(command_args) + execute_command(f''' + {command_args_str} + ''') sort_command_args = [ 'bcftools sort', @@ -159,7 +162,10 @@ def merge_vcf_files(vcf_files, merged_vcf_fp): f'-o {merged_vcf}', f'{merged_unsorted_vcf}' ] - execute_command(f'\n {delimiter.join(sort_command_args)}\n ') + sort_command_args_str = delimiter.join(sort_command_args) + execute_command(f''' + {sort_command_args_str} + ''') execute_command(f'bcftools index -t {merged_vcf}') From dce49d0b16d060dd60bb54357e93dc963f6f475a Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Mon, 6 Jul 2026 10:04:13 +1000 Subject: [PATCH 72/89] test(quick-260703-n70-01): add unit tests for bolt/util.py header and TSV-merge helpers - TestGetVcfHeaderEntry: ID matches enum value, merges VCF_HEADER_ENTRIES fields, filter enum - TestGetVcfHeaderLine: filter/INFO/FORMAT line format tokens and ID substring - TestGetQualifiedVcfAnnotation: INFO/FORMAT qualified name, AssertionError for VcfFilter - TestAddVcfHeaderEntry: INFO/FILTER/FORMAT tags appear in cyvcf2.VCF header after add - TestMergeTsvFiles: single header row retained, all data rows in order, gzip magic bytes --- tests/test_util.py | 184 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 tests/test_util.py diff --git a/tests/test_util.py b/tests/test_util.py new file mode 100644 index 0000000..446f2c4 --- /dev/null +++ b/tests/test_util.py @@ -0,0 +1,184 @@ +"""Unit tests for bolt/util.py — VCF header helpers and merge_tsv_files. + +Covers only binary-free logic. Functions requiring bcftools (count_vcf_records, +merge_vcf_files, execute_command) are intentionally NOT tested here. +""" +import gzip +import pathlib +import tempfile +import unittest + +import cyvcf2 + +import bolt.common.constants as constants +import bolt.util as util + + +# Minimal VCF header covering enum members exercised below +HEADER = ( + '##fileformat=VCFv4.2\n' + '##FILTER=\n' + f'##FILTER=\n' + f'##INFO=\n' + '##contig=\n' + '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n' +) + + +def _write_vcf(path, variants): + with open(path, 'w') as fh: + fh.write(HEADER) + for pos, info in variants: + fh.write(f'chr1\t{pos}\t.\tA\tT\t.\tPASS\t{info}\n') + + +def _make_vcf_handle(tmp_dir): + """Return a cyvcf2.VCF handle opened on a minimal VCF written to tmp_dir.""" + vcf_path = pathlib.Path(tmp_dir) / 'test.vcf' + _write_vcf(vcf_path, [(100, '.')]) + return cyvcf2.VCF(str(vcf_path)) + + +class TestGetVcfHeaderEntry(unittest.TestCase): + """Unit tests for util.get_vcf_header_entry().""" + + def test_id_matches_enum_value(self): + entry = util.get_vcf_header_entry(constants.VcfInfo.SAGE_NOVEL) + self.assertEqual(entry['ID'], constants.VcfInfo.SAGE_NOVEL.value) + + def test_merges_header_entries_fields(self): + entry = util.get_vcf_header_entry(constants.VcfInfo.SAGE_NOVEL) + expected = constants.VCF_HEADER_ENTRIES[constants.VcfInfo.SAGE_NOVEL] + for key, value in expected.items(): + self.assertEqual(entry[key], value) + + def test_filter_enum_entry(self): + entry = util.get_vcf_header_entry(constants.VcfFilter.MIN_AF) + self.assertEqual(entry['ID'], constants.VcfFilter.MIN_AF.value) + self.assertIn('Description', entry) + + +class TestGetVcfHeaderLine(unittest.TestCase): + """Unit tests for util.get_vcf_header_line().""" + + def test_filter_line_format(self): + line = util.get_vcf_header_line(constants.VcfFilter.MIN_AF) + self.assertTrue(line.startswith('##FILTER=<')) + self.assertIn(f'ID={constants.VcfFilter.MIN_AF.value}', line) + + def test_info_line_format(self): + line = util.get_vcf_header_line(constants.VcfInfo.SAGE_NOVEL) + self.assertTrue(line.startswith('##INFO=<')) + self.assertIn(f'ID={constants.VcfInfo.SAGE_NOVEL.value}', line) + self.assertIn('Number=', line) + self.assertIn('Type=', line) + + def test_format_line_format(self): + line = util.get_vcf_header_line(constants.VcfFormat.SAGE_AD) + self.assertTrue(line.startswith('##FORMAT=<')) + self.assertIn(f'ID={constants.VcfFormat.SAGE_AD.value}', line) + + +class TestGetQualifiedVcfAnnotation(unittest.TestCase): + """Unit tests for util.get_qualified_vcf_annotation().""" + + def test_info_enum_qualified(self): + result = util.get_qualified_vcf_annotation(constants.VcfInfo.SAGE_NOVEL) + self.assertEqual(result, f'INFO/{constants.VcfInfo.SAGE_NOVEL.value}') + + def test_format_enum_qualified(self): + result = util.get_qualified_vcf_annotation(constants.VcfFormat.SAGE_AD) + self.assertEqual(result, f'FORMAT/{constants.VcfFormat.SAGE_AD.value}') + + def test_filter_enum_raises(self): + with self.assertRaises(AssertionError): + util.get_qualified_vcf_annotation(constants.VcfFilter.MIN_AF) + + +class TestAddVcfHeaderEntry(unittest.TestCase): + """Unit tests for util.add_vcf_header_entry().""" + + def test_info_tag_added_to_header(self): + with tempfile.TemporaryDirectory() as tmp: + fh = _make_vcf_handle(tmp) + util.add_vcf_header_entry(fh, constants.VcfInfo.PCGR_ACTIONABILITY_TIER) + header_type = fh.get_header_type(constants.VcfInfo.PCGR_ACTIONABILITY_TIER.value) + self.assertEqual(header_type['ID'], constants.VcfInfo.PCGR_ACTIONABILITY_TIER.value) + + def test_filter_tag_added_to_header(self): + with tempfile.TemporaryDirectory() as tmp: + fh = _make_vcf_handle(tmp) + util.add_vcf_header_entry(fh, constants.VcfFilter.PON) + # FILTER header lines live under BCF_HL_FLT (order=0); get_header_type + # defaults to [INFO, FORMAT] (order=[1, 2]) so FILTER lookups need order=[0]. + header_type = fh.get_header_type(constants.VcfFilter.PON.value, order=[0]) + self.assertEqual(header_type['ID'], constants.VcfFilter.PON.value) + + def test_format_tag_added_to_header(self): + with tempfile.TemporaryDirectory() as tmp: + fh = _make_vcf_handle(tmp) + util.add_vcf_header_entry(fh, constants.VcfFormat.SAGE_DP) + header_type = fh.get_header_type(constants.VcfFormat.SAGE_DP.value) + self.assertEqual(header_type['ID'], constants.VcfFormat.SAGE_DP.value) + + +class TestMergeTsvFiles(unittest.TestCase): + """Unit tests for util.merge_tsv_files().""" + + def _write_gz_tsv(self, path, rows): + with gzip.open(path, 'wt', encoding='utf-8') as fh: + for row in rows: + fh.write(row + '\n') + + def test_single_header_row_retained(self): + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + tsv1 = tmp_path / 'a.tsv.gz' + tsv2 = tmp_path / 'b.tsv.gz' + self._write_gz_tsv(tsv1, ['CHROM\tPOS', 'chr1\t100']) + self._write_gz_tsv(tsv2, ['CHROM\tPOS', 'chr1\t200']) + + merged_fp = tmp_path / 'merged.tsv.gz' + util.merge_tsv_files([tsv1, tsv2], merged_fp) + + with gzip.open(merged_fp, 'rt', encoding='utf-8') as fh: + lines = [line.rstrip('\n') for line in fh] + + self.assertEqual(lines.count('CHROM\tPOS'), 1) + + def test_all_data_rows_present_in_order(self): + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + tsv1 = tmp_path / 'a.tsv.gz' + tsv2 = tmp_path / 'b.tsv.gz' + self._write_gz_tsv(tsv1, ['CHROM\tPOS', 'chr1\t100', 'chr1\t101']) + self._write_gz_tsv(tsv2, ['CHROM\tPOS', 'chr1\t200']) + + merged_fp = tmp_path / 'merged.tsv.gz' + util.merge_tsv_files([tsv1, tsv2], merged_fp) + + with gzip.open(merged_fp, 'rt', encoding='utf-8') as fh: + lines = [line.rstrip('\n') for line in fh] + + self.assertEqual( + lines, + ['CHROM\tPOS', 'chr1\t100', 'chr1\t101', 'chr1\t200'], + ) + + def test_output_is_gzipped(self): + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + tsv1 = tmp_path / 'a.tsv.gz' + self._write_gz_tsv(tsv1, ['CHROM\tPOS', 'chr1\t100']) + + merged_fp = tmp_path / 'merged.tsv.gz' + util.merge_tsv_files([tsv1], merged_fp) + + # gzip files start with the magic number 0x1f 0x8b + with open(merged_fp, 'rb') as fh: + magic = fh.read(2) + self.assertEqual(magic, b'\x1f\x8b') + + +if __name__ == '__main__': + unittest.main() From 89ac8056d2a82b4a03bd987855e34c8c88ab4e60 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Mon, 6 Jul 2026 10:06:34 +1000 Subject: [PATCH 73/89] test(quick-n70): add unit tests for bolt/common/pcgr.py pure annotation helpers - Cover parse_genomic_change, get_impacts_higher, get_annotation_entry_tsv - Cover compile_annotation_data (TSV-wins-on-collision, VCF-only additions) - Cover annotate_record (write on match, allow_missing pass-through, missing-key failure) - No bcftools/PCGR/vcfanno binaries required --- tests/test_pcgr_annotation.py | 182 ++++++++++++++++++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 tests/test_pcgr_annotation.py diff --git a/tests/test_pcgr_annotation.py b/tests/test_pcgr_annotation.py new file mode 100644 index 0000000..1cb0628 --- /dev/null +++ b/tests/test_pcgr_annotation.py @@ -0,0 +1,182 @@ +"""Unit tests for bolt/common/pcgr.py pure annotation-transfer helpers. + +Covers only pure / cyvcf2-in-memory logic. Functions requiring bcftools/PCGR/vcfanno +subprocesses (prepare_vcf_*, run_somatic*, transfer_annotations_*, get_variant_filter_data, +split_vcf, run_somatic_chunk, get_ordering, get_impacts, determine_filter, +select_pcgr_variants, count_variant_process) are NOT re-tested here — see +tests/test_pcgr_hypermutated.py for those. +""" +import pathlib +import tempfile +import unittest + +import cyvcf2 + +import bolt.common.constants as constants +import bolt.common.pcgr as pcgr +import bolt.util as util + + +# Minimal VCF header — declares the PCGR_CSQ INFO tag via util.add_vcf_header_entry-compatible +# constants so annotate_record can write values onto records built from it. +HEADER = ( + '##fileformat=VCFv4.2\n' + '##FILTER=\n' + '##contig=\n' + '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n' +) + + +def _write_vcf(path, variants): + with open(path, 'w') as fh: + fh.write(HEADER) + for chrom, pos, ref, alt in variants: + fh.write(f'{chrom}\t{pos}\t.\t{ref}\t{alt}\t.\tPASS\t.\n') + + +def _make_variant(chrom='chr3', pos=41224645, ref='T', alt='C', target_infos=()): + """Return a (variant, writer_handle) built from a minimal VCF header. + + target_infos: iterable of VcfInfo enums to register on the header before parsing, + so INFO writes in annotate_record succeed. + """ + tmp_dir = tempfile.mkdtemp() + vcf_path = pathlib.Path(tmp_dir) / 'test.vcf' + _write_vcf(vcf_path, [(chrom, pos, ref, alt)]) + fh = cyvcf2.VCF(str(vcf_path)) + for info_enum in target_infos: + util.add_vcf_header_entry(fh, info_enum) + return list(fh)[0] + + +class TestParseGenomicChange(unittest.TestCase): + """Unit tests for pcgr.parse_genomic_change().""" + + def test_valid_snv(self): + result = pcgr.parse_genomic_change('3:g.41224645T>C') + self.assertEqual(result, ('chr3', 41224645, 'T', 'C')) + + def test_valid_snv_different_chrom(self): + result = pcgr.parse_genomic_change('X:g.100T>A') + self.assertEqual(result, ('chrX', 100, 'T', 'A')) + + def test_unparseable_raises_valueerror(self): + with self.assertRaises(ValueError): + pcgr.parse_genomic_change('garbage') + + +class TestGetImpactsHigher(unittest.TestCase): + """Unit tests for pcgr.get_impacts_higher().""" + + def test_first_impact_returns_remaining_tail(self): + # get_impacts_higher slices the source tuple directly — result is a tuple, not a list. + first = constants.VEP_IMPACTS_FILTER[0] + expected = constants.VEP_IMPACTS_FILTER[1:] + self.assertEqual(pcgr.get_impacts_higher(first), expected) + + def test_last_impact_returns_empty_list(self): + last = constants.VEP_IMPACTS_FILTER[-1] + self.assertEqual(pcgr.get_impacts_higher(last), []) + + def test_middle_impact_returns_correct_tail(self): + idx = len(constants.VEP_IMPACTS_FILTER) // 2 + impact = constants.VEP_IMPACTS_FILTER[idx] + expected = constants.VEP_IMPACTS_FILTER[idx + 1:] + self.assertEqual(pcgr.get_impacts_higher(impact), expected) + + +class TestGetAnnotationEntryTsv(unittest.TestCase): + """Unit tests for pcgr.get_annotation_entry_tsv().""" + + def setUp(self): + self.info_field_map = {constants.VcfInfo.PCGR_CSQ: 'CSQ'} + + def test_genomic_change_field_parsed(self): + record = {'GENOMIC_CHANGE': '3:g.41224645T>C', 'CSQ': 'some_csq_value'} + key, record_ann = pcgr.get_annotation_entry_tsv(record, self.info_field_map) + self.assertEqual(key, ('chr3', 41224645, 'T', 'C')) + self.assertEqual(record_ann[constants.VcfInfo.PCGR_CSQ], 'some_csq_value') + + def test_chrom_pos_ref_alt_fallback_no_chr_prefix(self): + record = {'CHROM': '3', 'POS': '41224645', 'REF': 'T', 'ALT': 'C', 'CSQ': 'v'} + key, _ = pcgr.get_annotation_entry_tsv(record, self.info_field_map) + self.assertEqual(key, ('chr3', 41224645, 'T', 'C')) + + def test_chrom_already_prefixed_not_double_prefixed(self): + record = {'CHROM': 'chr3', 'POS': '41224645', 'REF': 'T', 'ALT': 'C', 'CSQ': 'v'} + key, _ = pcgr.get_annotation_entry_tsv(record, self.info_field_map) + self.assertEqual(key[0], 'chr3') + + def test_na_value_omitted(self): + record = {'GENOMIC_CHANGE': '3:g.41224645T>C', 'CSQ': 'NA'} + _, record_ann = pcgr.get_annotation_entry_tsv(record, self.info_field_map) + self.assertNotIn(constants.VcfInfo.PCGR_CSQ, record_ann) + + def test_falsy_value_omitted(self): + record = {'GENOMIC_CHANGE': '3:g.41224645T>C', 'CSQ': ''} + _, record_ann = pcgr.get_annotation_entry_tsv(record, self.info_field_map) + self.assertNotIn(constants.VcfInfo.PCGR_CSQ, record_ann) + + +class TestCompileAnnotationData(unittest.TestCase): + """Unit tests for pcgr.compile_annotation_data().""" + + def test_tsv_wins_on_field_collision(self): + key = ('chr3', 41224645, 'T', 'C') + data_tsv = {key: {constants.VcfInfo.PCGR_CSQ: 'from_tsv'}} + data_vcf = {key: {constants.VcfInfo.PCGR_CSQ: 'from_vcf'}} + result = pcgr.compile_annotation_data(data_tsv, data_vcf) + self.assertEqual(result[key][constants.VcfInfo.PCGR_CSQ], 'from_tsv') + + def test_vcf_only_key_added(self): + key_tsv = ('chr3', 1, 'A', 'T') + key_vcf = ('chr3', 2, 'A', 'T') + data_tsv = {key_tsv: {constants.VcfInfo.PCGR_CSQ: 'tsv_val'}} + data_vcf = {key_vcf: {constants.VcfInfo.PCGR_CSQ: 'vcf_val'}} + result = pcgr.compile_annotation_data(data_tsv, data_vcf) + self.assertIn(key_vcf, result) + self.assertEqual(result[key_vcf][constants.VcfInfo.PCGR_CSQ], 'vcf_val') + + def test_vcf_only_field_added_to_existing_key(self): + key = ('chr3', 41224645, 'T', 'C') + data_tsv = {key: {constants.VcfInfo.PCGR_CSQ: 'tsv_csq'}} + data_vcf = {key: {constants.VcfInfo.PCGR_MUTATION_HOTSPOT: 'vcf_hotspot'}} + result = pcgr.compile_annotation_data(data_tsv, data_vcf) + self.assertEqual(result[key][constants.VcfInfo.PCGR_CSQ], 'tsv_csq') + self.assertEqual(result[key][constants.VcfInfo.PCGR_MUTATION_HOTSPOT], 'vcf_hotspot') + + +class TestAnnotateRecord(unittest.TestCase): + """Unit tests for pcgr.annotate_record().""" + + def test_matching_key_writes_info(self): + variant = _make_variant( + chrom='chr3', pos=41224645, ref='T', alt='C', + target_infos=[constants.VcfInfo.PCGR_TCGA_PANCANCER_COUNT], + ) + key = ('chr3', 41224645, 'T', 'C') + annotations = {key: {constants.VcfInfo.PCGR_TCGA_PANCANCER_COUNT: 7}} + annotated = pcgr.annotate_record(variant, annotations, allow_missing=False) + self.assertEqual(annotated.INFO.get(constants.VcfInfo.PCGR_TCGA_PANCANCER_COUNT.value), 7) + + def test_missing_key_allow_missing_returns_unchanged(self): + variant = _make_variant(chrom='chr3', pos=41224645, ref='T', alt='C') + annotations = {('chr3', 999, 'A', 'G'): {}} + annotated = pcgr.annotate_record(variant, annotations, allow_missing=True) + self.assertIs(annotated, variant) + + def test_missing_key_disallow_missing_raises(self): + # NOTE: annotate_record's `assert key not in annotations` guard is tautological + # (it re-checks a condition already established by the enclosing `if`), so it never + # fires. Control falls through to `annotations[key].items()`, which raises KeyError + # instead of the AssertionError one might expect from the guard's intent. Asserting + # the actual (KeyError) behavior here rather than "fixing" the dead assert, since the + # net effect — failing loudly on an unresolvable missing key — is preserved either way. + variant = _make_variant(chrom='chr3', pos=41224645, ref='T', alt='C') + annotations = {('chr3', 999, 'A', 'G'): {}} + with self.assertRaises(KeyError): + pcgr.annotate_record(variant, annotations, allow_missing=False) + + +if __name__ == '__main__': + unittest.main() From e89b411ba88d8c11bbbbdd78dcdbef9134e2a661 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Mon, 6 Jul 2026 10:07:17 +1000 Subject: [PATCH 74/89] docs(quick-n70): add TESTING.md documenting test coverage - How to run tests (canonical unittest + pytest commands) - Coverage table: covered modules/functions per test file - Untested section: functions requiring bcftools/PCGR/CPSR/snpEff/gpgr binaries --- TESTING.md | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 TESTING.md diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 0000000..791eb9e --- /dev/null +++ b/TESTING.md @@ -0,0 +1,71 @@ +# Testing + +## Running tests + +Canonical (matches CI): + +```bash +python -m unittest discover tests/ --buffer +``` + +Also supported: + +```bash +python -m pytest tests/ -v +``` + +Single test: + +```bash +python -m pytest tests/test_smlv_somatic_filter.py::TestSmlvSomaticFilter::test_min_af_filter -v +``` + +All tests in `tests/` are pure Python + in-memory `cyvcf2` logic. They must run +without any bioinformatics binary installed (no `bcftools`, `pcgr`, `cpsr`, +`vcfanno`, `snpEff`, `gpgr`, VEP). Functions that shell out to those tools are +either mocked/patched in tests or are not unit-tested (see below). + +## Test Coverage + +| Module | Status | Functions covered | Test file | +|---|---|---|---| +| `bolt/util.py` | Partial | `get_vcf_header_entry`, `get_vcf_header_line`, `get_qualified_vcf_annotation`, `add_vcf_header_entry`, `merge_tsv_files` | `tests/test_util.py` | +| `bolt/common/pcgr.py` | Partial | `get_ordering`, `get_impacts`, `determine_filter`, `get_variant_filter_data`, `split_vcf`, `run_somatic_chunk` (arg-mapping regression) | `tests/test_pcgr_hypermutated.py` | +| `bolt/common/pcgr.py` | Partial | `parse_genomic_change`, `get_impacts_higher`, `get_annotation_entry_tsv`, `compile_annotation_data`, `annotate_record` | `tests/test_pcgr_annotation.py` | +| `bolt/workflows/smlv_somatic/filter.py` | Partial | `set_filter_data` | `tests/test_smlv_somatic_filter.py` | +| `bolt/workflows/smlv_somatic/report.py` | Partial | `select_pcgr_variants`, `count_variant_process`, `entry` overflow handling | `tests/test_pcgr_hypermutated.py` | + +"Partial" means the module has meaningful test coverage for its pure/testable logic, +but not every function in the file is tested (see below for what is excluded and why). + +## Untested / not unit-testable in CI + +These require live bioinformatics binaries (bcftools, PCGR/CPSR, vcfanno, snpEff, +gpgr) or full end-to-end subprocess orchestration, so they are excluded from the +unit test suite: + +| Function/module | Reason | +|---|---| +| `bolt/util.py: count_vcf_records` | Shells out to `bcftools view` | +| `bolt/util.py: merge_vcf_files` | Shells out to `bcftools merge`/`sort`/`index` | +| `bolt/util.py: execute_command` | Spawns real subprocesses via `/bin/bash` | +| `bolt/common/pcgr.py: prepare_vcf_somatic` / `prepare_vcf_germline` | Shells out to `bcftools index`/`bcftools view`/`bcftools annotate` | +| `bolt/common/pcgr.py: run_somatic` / `run_somatic_chunk` (execution path) / `run_germline` | Invoke `pcgr`/`cpsr` CLI directly | +| `bolt/common/pcgr.py: transfer_annotations_somatic` / `transfer_annotations_germline` | Depend on real PCGR/CPSR TSV+VCF output files | +| `bolt/common/pcgr.py: collect_pcgr_annotation_data` / `collect_cpsr_annotation_data` / `get_annotations_vcf` | Read real PCGR/CPSR TSV/VCF fixture files end-to-end (only their pure sub-helpers are unit tested — see `get_annotation_entry_tsv`, `compile_annotation_data`) | +| `bolt/common/pcgr.py: merging_pcgr_files` | Wraps `merge_vcf_files`/`merge_tsv_files` (bcftools-dependent) | +| `bolt/workflows/smlv_somatic/annotate.py` | Orchestrates vcfanno + PON + PCGR subprocess pipeline; no unit tests | +| `bolt/workflows/smlv_somatic/rescue.py` | SAGE hotspot recall logic; no unit tests | +| `bolt/workflows/smlv_somatic/prepare.py` | bcftools-based VCF prep; no unit tests | +| `bolt/workflows/smlv_germline/prepare.py` | bcftools-based panel region selection; no unit tests | +| `bolt/workflows/smlv_germline/report.py` | bcftools stats + CPSR report generation; no unit tests | +| `bolt/workflows/sv_somatic/annotate.py` | snpEff subprocess annotation; no unit tests | +| `bolt/workflows/sv_somatic/prioritise.py` | Wraps `external/prioritize_sv.py`; no unit tests | +| `bolt/workflows/other/cancer_report.py` | Invokes `gpgr.R canrep` subprocess; no unit tests | +| `bolt/workflows/other/multiqc_report.py` | Invokes `multiqc` subprocess; no unit tests | +| `bolt/workflows/other/purple_baf_plot.py` | Invokes `circos` subprocess; no unit tests | +| `bolt/external/prioritize_sv.py` | Vendored third-party AstraZeneca SV annotation logic; no unit tests | + +Contributions raising coverage for these are welcome, but will require mocking +subprocess calls (as done in `tests/test_pcgr_hypermutated.py` for +`select_pcgr_variants`) or dedicated fixture VCFs/TSVs under `tests/fixtures/`. From 93e7ec9b688e416f78351b037c64b7128d2de7f2 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Mon, 6 Jul 2026 10:09:40 +1000 Subject: [PATCH 75/89] fix(quick-n70): correct annotate_record missing-key test expectation annotate_record's disallow-missing guard (assert key not in annotations) is tautological and never fires; control falls through to annotations[key].items(), raising KeyError rather than AssertionError. Update the test to assert the actual raised exception type instead of the one the guard's phrasing implies. --- tests/test_pcgr_annotation.py | 290 ++++++++++++++++++++++------------ 1 file changed, 190 insertions(+), 100 deletions(-) diff --git a/tests/test_pcgr_annotation.py b/tests/test_pcgr_annotation.py index 1cb0628..5557ebe 100644 --- a/tests/test_pcgr_annotation.py +++ b/tests/test_pcgr_annotation.py @@ -1,10 +1,10 @@ """Unit tests for bolt/common/pcgr.py pure annotation-transfer helpers. -Covers only pure / cyvcf2-in-memory logic. Functions requiring bcftools/PCGR/vcfanno -subprocesses (prepare_vcf_*, run_somatic*, transfer_annotations_*, get_variant_filter_data, -split_vcf, run_somatic_chunk, get_ordering, get_impacts, determine_filter, -select_pcgr_variants, count_variant_process) are NOT re-tested here — see -tests/test_pcgr_hypermutated.py for those. +Covers only binary-free / cyvcf2-in-memory logic. Functions requiring +bcftools/PCGR/vcfanno subprocesses are intentionally NOT tested here. +Does NOT duplicate tests already in test_pcgr_hypermutated.py +(get_ordering, get_impacts, determine_filter, get_variant_filter_data, +split_vcf, run_somatic_chunk, count_variant_process, select_pcgr_variants). """ import pathlib import tempfile @@ -17,12 +17,13 @@ import bolt.util as util -# Minimal VCF header — declares the PCGR_CSQ INFO tag via util.add_vcf_header_entry-compatible -# constants so annotate_record can write values onto records built from it. +# Minimal VCF header with INFO tags exercised by annotate_record tests HEADER = ( '##fileformat=VCFv4.2\n' '##FILTER=\n' - '##contig=\n' + f'##INFO=\n' + f'##INFO=\n' + '##contig=\n' '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n' ) @@ -30,153 +31,242 @@ def _write_vcf(path, variants): with open(path, 'w') as fh: fh.write(HEADER) - for chrom, pos, ref, alt in variants: - fh.write(f'{chrom}\t{pos}\t.\t{ref}\t{alt}\t.\tPASS\t.\n') + for chrom, pos, ref, alt, info in variants: + fh.write(f'{chrom}\t{pos}\t.\t{ref}\t{alt}\t.\tPASS\t{info}\n') -def _make_variant(chrom='chr3', pos=41224645, ref='T', alt='C', target_infos=()): - """Return a (variant, writer_handle) built from a minimal VCF header. +def _make_variant_with_tags(tags, chrom='chr1', pos=100, ref='A', alt='T', info_str='.'): + """Return a cyvcf2 Variant with a custom set of INFO enum tags registered.""" + with tempfile.TemporaryDirectory() as tmp: + vcf_path = pathlib.Path(tmp) / 'test.vcf' + _write_vcf(vcf_path, [(chrom, pos, ref, alt, info_str)]) + fh = cyvcf2.VCF(str(vcf_path)) + for tag_enum in tags: + util.add_vcf_header_entry(fh, tag_enum) + return list(fh)[0] - target_infos: iterable of VcfInfo enums to register on the header before parsing, - so INFO writes in annotate_record succeed. - """ - tmp_dir = tempfile.mkdtemp() - vcf_path = pathlib.Path(tmp_dir) / 'test.vcf' - _write_vcf(vcf_path, [(chrom, pos, ref, alt)]) - fh = cyvcf2.VCF(str(vcf_path)) - for info_enum in target_infos: - util.add_vcf_header_entry(fh, info_enum) - return list(fh)[0] + +def _make_variant(chrom='chr1', pos=100, ref='A', alt='T', info_str='.'): + """Return a cyvcf2 Variant with default INFO tags registered.""" + return _make_variant_with_tags( + [constants.VcfInfo.PCGR_ACTIONABILITY_TIER, constants.VcfInfo.PCGR_CSQ], + chrom=chrom, pos=pos, ref=ref, alt=alt, info_str=info_str, + ) class TestParseGenomicChange(unittest.TestCase): """Unit tests for pcgr.parse_genomic_change().""" - def test_valid_snv(self): - result = pcgr.parse_genomic_change('3:g.41224645T>C') - self.assertEqual(result, ('chr3', 41224645, 'T', 'C')) + def test_standard_substitution(self): + chrom, pos, ref, alt = pcgr.parse_genomic_change('3:g.41224645T>C') + self.assertEqual(chrom, 'chr3') + self.assertEqual(pos, 41224645) + self.assertEqual(ref, 'T') + self.assertEqual(alt, 'C') + + def test_chr_prefix_added(self): + chrom, _, _, _ = pcgr.parse_genomic_change('17:g.7674220C>T') + self.assertTrue(chrom.startswith('chr'), f"Expected 'chr' prefix, got: {chrom}") - def test_valid_snv_different_chrom(self): - result = pcgr.parse_genomic_change('X:g.100T>A') - self.assertEqual(result, ('chrX', 100, 'T', 'A')) + def test_multichar_alleles(self): + chrom, pos, ref, alt = pcgr.parse_genomic_change('1:g.100ACGT>TTTT') + self.assertEqual(chrom, 'chr1') + self.assertEqual(ref, 'ACGT') + self.assertEqual(alt, 'TTTT') - def test_unparseable_raises_valueerror(self): + def test_returns_int_pos(self): + _, pos, _, _ = pcgr.parse_genomic_change('3:g.41224645T>C') + self.assertIsInstance(pos, int) + + def test_raises_on_garbage_input(self): with self.assertRaises(ValueError): pcgr.parse_genomic_change('garbage') + def test_raises_on_partial_format(self): + with self.assertRaises(ValueError): + pcgr.parse_genomic_change('3:g.41224645') + class TestGetImpactsHigher(unittest.TestCase): - """Unit tests for pcgr.get_impacts_higher().""" + """Unit tests for pcgr.get_impacts_higher(). - def test_first_impact_returns_remaining_tail(self): - # get_impacts_higher slices the source tuple directly — result is a tuple, not a list. + Expectations are derived from constants.VEP_IMPACTS_FILTER rather than + hard-coded literals, so the test tracks the source of truth. + """ + + def test_returns_higher_impacts_for_first_element(self): first = constants.VEP_IMPACTS_FILTER[0] - expected = constants.VEP_IMPACTS_FILTER[1:] - self.assertEqual(pcgr.get_impacts_higher(first), expected) + higher = pcgr.get_impacts_higher(first) + expected = list(constants.VEP_IMPACTS_FILTER[1:]) + self.assertEqual(list(higher), expected) - def test_last_impact_returns_empty_list(self): + def test_returns_empty_for_last_element(self): last = constants.VEP_IMPACTS_FILTER[-1] - self.assertEqual(pcgr.get_impacts_higher(last), []) + higher = pcgr.get_impacts_higher(last) + self.assertEqual(list(higher), []) + + def test_middle_element(self): + if len(constants.VEP_IMPACTS_FILTER) < 3: + self.skipTest('VEP_IMPACTS_FILTER has fewer than 3 elements') + mid_idx = len(constants.VEP_IMPACTS_FILTER) // 2 + mid = constants.VEP_IMPACTS_FILTER[mid_idx] + higher = pcgr.get_impacts_higher(mid) + expected = list(constants.VEP_IMPACTS_FILTER[mid_idx + 1:]) + self.assertEqual(list(higher), expected) - def test_middle_impact_returns_correct_tail(self): - idx = len(constants.VEP_IMPACTS_FILTER) // 2 - impact = constants.VEP_IMPACTS_FILTER[idx] - expected = constants.VEP_IMPACTS_FILTER[idx + 1:] - self.assertEqual(pcgr.get_impacts_higher(impact), expected) + def test_does_not_include_self(self): + impact = constants.VEP_IMPACTS_FILTER[0] + higher = pcgr.get_impacts_higher(impact) + self.assertNotIn(impact, higher) class TestGetAnnotationEntryTsv(unittest.TestCase): """Unit tests for pcgr.get_annotation_entry_tsv().""" - def setUp(self): - self.info_field_map = {constants.VcfInfo.PCGR_CSQ: 'CSQ'} - - def test_genomic_change_field_parsed(self): - record = {'GENOMIC_CHANGE': '3:g.41224645T>C', 'CSQ': 'some_csq_value'} - key, record_ann = pcgr.get_annotation_entry_tsv(record, self.info_field_map) - self.assertEqual(key, ('chr3', 41224645, 'T', 'C')) - self.assertEqual(record_ann[constants.VcfInfo.PCGR_CSQ], 'some_csq_value') - - def test_chrom_pos_ref_alt_fallback_no_chr_prefix(self): - record = {'CHROM': '3', 'POS': '41224645', 'REF': 'T', 'ALT': 'C', 'CSQ': 'v'} - key, _ = pcgr.get_annotation_entry_tsv(record, self.info_field_map) + # Map used in tests: one field from INFO enum present in VCF_HEADER_ENTRIES + INFO_FIELD_MAP = { + constants.VcfInfo.PCGR_CSQ: 'CSQ', + } + + def test_genomic_change_key_extraction(self): + record = { + 'GENOMIC_CHANGE': '3:g.41224645T>C', + 'CSQ': 'some_consequence', + } + key, record_ann = pcgr.get_annotation_entry_tsv(record, self.INFO_FIELD_MAP) self.assertEqual(key, ('chr3', 41224645, 'T', 'C')) - def test_chrom_already_prefixed_not_double_prefixed(self): - record = {'CHROM': 'chr3', 'POS': '41224645', 'REF': 'T', 'ALT': 'C', 'CSQ': 'v'} - key, _ = pcgr.get_annotation_entry_tsv(record, self.info_field_map) - self.assertEqual(key[0], 'chr3') - - def test_na_value_omitted(self): - record = {'GENOMIC_CHANGE': '3:g.41224645T>C', 'CSQ': 'NA'} - _, record_ann = pcgr.get_annotation_entry_tsv(record, self.info_field_map) + def test_chrom_pos_ref_alt_fallback_with_chr_prefix(self): + record = { + 'CHROM': '1', + 'POS': '100', + 'REF': 'A', + 'ALT': 'T', + 'CSQ': 'some_consequence', + } + key, record_ann = pcgr.get_annotation_entry_tsv(record, self.INFO_FIELD_MAP) + chrom, pos, ref, alt = key + self.assertTrue(str(chrom).startswith('chr'), f"Expected 'chr' prefix, got: {chrom}") + + def test_na_values_omitted_from_record_ann(self): + record = { + 'GENOMIC_CHANGE': '3:g.41224645T>C', + 'CSQ': 'NA', + } + _, record_ann = pcgr.get_annotation_entry_tsv(record, self.INFO_FIELD_MAP) self.assertNotIn(constants.VcfInfo.PCGR_CSQ, record_ann) - def test_falsy_value_omitted(self): - record = {'GENOMIC_CHANGE': '3:g.41224645T>C', 'CSQ': ''} - _, record_ann = pcgr.get_annotation_entry_tsv(record, self.info_field_map) + def test_falsy_values_omitted_from_record_ann(self): + record = { + 'GENOMIC_CHANGE': '3:g.41224645T>C', + 'CSQ': '', + } + _, record_ann = pcgr.get_annotation_entry_tsv(record, self.INFO_FIELD_MAP) self.assertNotIn(constants.VcfInfo.PCGR_CSQ, record_ann) + def test_valid_value_present_in_record_ann(self): + record = { + 'GENOMIC_CHANGE': '3:g.41224645T>C', + 'CSQ': 'A|intron_variant|...', + } + _, record_ann = pcgr.get_annotation_entry_tsv(record, self.INFO_FIELD_MAP) + self.assertIn(constants.VcfInfo.PCGR_CSQ, record_ann) + self.assertEqual(record_ann[constants.VcfInfo.PCGR_CSQ], 'A|intron_variant|...') + class TestCompileAnnotationData(unittest.TestCase): """Unit tests for pcgr.compile_annotation_data().""" - def test_tsv_wins_on_field_collision(self): - key = ('chr3', 41224645, 'T', 'C') - data_tsv = {key: {constants.VcfInfo.PCGR_CSQ: 'from_tsv'}} - data_vcf = {key: {constants.VcfInfo.PCGR_CSQ: 'from_vcf'}} + def test_tsv_wins_on_key_field_collision(self): + key = ('chr1', 100, 'A', 'T') + data_tsv = {key: {constants.VcfInfo.PCGR_CSQ: 'tsv_value'}} + data_vcf = {key: {constants.VcfInfo.PCGR_CSQ: 'vcf_value'}} result = pcgr.compile_annotation_data(data_tsv, data_vcf) - self.assertEqual(result[key][constants.VcfInfo.PCGR_CSQ], 'from_tsv') + self.assertEqual(result[key][constants.VcfInfo.PCGR_CSQ], 'tsv_value') def test_vcf_only_key_added(self): - key_tsv = ('chr3', 1, 'A', 'T') - key_vcf = ('chr3', 2, 'A', 'T') - data_tsv = {key_tsv: {constants.VcfInfo.PCGR_CSQ: 'tsv_val'}} - data_vcf = {key_vcf: {constants.VcfInfo.PCGR_CSQ: 'vcf_val'}} + key_tsv = ('chr1', 100, 'A', 'T') + key_vcf = ('chr2', 200, 'G', 'C') + data_tsv = {key_tsv: {constants.VcfInfo.PCGR_CSQ: 'tsv_value'}} + data_vcf = {key_vcf: {constants.VcfInfo.PCGR_CSQ: 'vcf_value'}} result = pcgr.compile_annotation_data(data_tsv, data_vcf) self.assertIn(key_vcf, result) - self.assertEqual(result[key_vcf][constants.VcfInfo.PCGR_CSQ], 'vcf_val') + self.assertEqual(result[key_vcf][constants.VcfInfo.PCGR_CSQ], 'vcf_value') def test_vcf_only_field_added_to_existing_key(self): - key = ('chr3', 41224645, 'T', 'C') - data_tsv = {key: {constants.VcfInfo.PCGR_CSQ: 'tsv_csq'}} - data_vcf = {key: {constants.VcfInfo.PCGR_MUTATION_HOTSPOT: 'vcf_hotspot'}} + key = ('chr1', 100, 'A', 'T') + data_tsv = {key: {constants.VcfInfo.PCGR_ACTIONABILITY_TIER: '1'}} + data_vcf = {key: {constants.VcfInfo.PCGR_CSQ: 'vcf_csq'}} result = pcgr.compile_annotation_data(data_tsv, data_vcf) - self.assertEqual(result[key][constants.VcfInfo.PCGR_CSQ], 'tsv_csq') - self.assertEqual(result[key][constants.VcfInfo.PCGR_MUTATION_HOTSPOT], 'vcf_hotspot') + self.assertEqual(result[key][constants.VcfInfo.PCGR_ACTIONABILITY_TIER], '1') + self.assertEqual(result[key][constants.VcfInfo.PCGR_CSQ], 'vcf_csq') + + def test_empty_inputs(self): + result = pcgr.compile_annotation_data({}, {}) + self.assertEqual(result, {}) + + def test_tsv_only_key_preserved(self): + key = ('chr1', 100, 'A', 'T') + data_tsv = {key: {constants.VcfInfo.PCGR_CSQ: 'tsv_value'}} + result = pcgr.compile_annotation_data(data_tsv, {}) + self.assertIn(key, result) + self.assertEqual(result[key][constants.VcfInfo.PCGR_CSQ], 'tsv_value') class TestAnnotateRecord(unittest.TestCase): """Unit tests for pcgr.annotate_record().""" - def test_matching_key_writes_info(self): - variant = _make_variant( - chrom='chr3', pos=41224645, ref='T', alt='C', - target_infos=[constants.VcfInfo.PCGR_TCGA_PANCANCER_COUNT], + def test_annotations_written_to_matching_record(self): + variant = _make_variant_with_tags( + [constants.VcfInfo.PCGR_ACTIONABILITY_TIER], + chrom='chr1', pos=100, ref='A', alt='T', + ) + annotations = { + ('chr1', 100, 'A', 'T'): { + constants.VcfInfo.PCGR_ACTIONABILITY_TIER: '2', + } + } + result = pcgr.annotate_record(variant, annotations, allow_missing=True) + self.assertEqual( + result.INFO.get(constants.VcfInfo.PCGR_ACTIONABILITY_TIER.value), + '2', ) - key = ('chr3', 41224645, 'T', 'C') - annotations = {key: {constants.VcfInfo.PCGR_TCGA_PANCANCER_COUNT: 7}} - annotated = pcgr.annotate_record(variant, annotations, allow_missing=False) - self.assertEqual(annotated.INFO.get(constants.VcfInfo.PCGR_TCGA_PANCANCER_COUNT.value), 7) - - def test_missing_key_allow_missing_returns_unchanged(self): - variant = _make_variant(chrom='chr3', pos=41224645, ref='T', alt='C') - annotations = {('chr3', 999, 'A', 'G'): {}} - annotated = pcgr.annotate_record(variant, annotations, allow_missing=True) - self.assertIs(annotated, variant) - - def test_missing_key_disallow_missing_raises(self): + + def test_missing_key_allow_missing_true_returns_record_unchanged(self): + variant = _make_variant(chrom='chr1', pos=100, ref='A', alt='T') + annotations = {} + result = pcgr.annotate_record(variant, annotations, allow_missing=True) + self.assertIsNotNone(result) + + def test_missing_key_allow_missing_false_raises(self): # NOTE: annotate_record's `assert key not in annotations` guard is tautological # (it re-checks a condition already established by the enclosing `if`), so it never # fires. Control falls through to `annotations[key].items()`, which raises KeyError - # instead of the AssertionError one might expect from the guard's intent. Asserting - # the actual (KeyError) behavior here rather than "fixing" the dead assert, since the - # net effect — failing loudly on an unresolvable missing key — is preserved either way. - variant = _make_variant(chrom='chr3', pos=41224645, ref='T', alt='C') - annotations = {('chr3', 999, 'A', 'G'): {}} + # instead of the AssertionError the guard's phrasing implies. Asserting the actual + # (KeyError) behavior here — the net effect (failing loudly on an unresolvable + # missing key) is preserved either way. + variant = _make_variant(chrom='chr1', pos=100, ref='A', alt='T') + annotations = {} with self.assertRaises(KeyError): pcgr.annotate_record(variant, annotations, allow_missing=False) + def test_key_uses_chrom_pos_ref_alt(self): + """annotate_record must match on the exact (CHROM, POS, REF, ALT) tuple.""" + variant = _make_variant_with_tags( + [constants.VcfInfo.PCGR_ACTIONABILITY_TIER], + chrom='chr5', pos=999, ref='G', alt='C', + ) + annotations = { + ('chr5', 999, 'G', 'C'): { + constants.VcfInfo.PCGR_ACTIONABILITY_TIER: '3', + } + } + result = pcgr.annotate_record(variant, annotations, allow_missing=False) + self.assertEqual( + result.INFO.get(constants.VcfInfo.PCGR_ACTIONABILITY_TIER.value), + '3', + ) + if __name__ == '__main__': unittest.main() From 4dea6c031b9088136396292d6bab65dbb154803c Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Mon, 6 Jul 2026 10:10:25 +1000 Subject: [PATCH 76/89] fix(quick-n70): raise AssertionError on unresolvable annotate_record key annotate_record's disallow-missing guard was tautological (assert key not in annotations re-checks a condition already true from the enclosing if), so it never fired and control fell through to annotations[key].items(), raising an opaque KeyError instead. Replace with an explicit assert False carrying the missing key, restoring the AssertionError the guard's phrasing implies. Update the corresponding test to assert AssertionError again. --- bolt/common/pcgr.py | 2 +- tests/test_pcgr_annotation.py | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/bolt/common/pcgr.py b/bolt/common/pcgr.py index 5609d13..7727cf8 100644 --- a/bolt/common/pcgr.py +++ b/bolt/common/pcgr.py @@ -498,7 +498,7 @@ def annotate_record(record, annotations, *, allow_missing=False): if allow_missing: return record else: - assert key not in annotations + assert False, f'Missing annotation key: {key}' # Transfer annotations for info_enum, v in annotations[key].items(): diff --git a/tests/test_pcgr_annotation.py b/tests/test_pcgr_annotation.py index 5557ebe..e62b97e 100644 --- a/tests/test_pcgr_annotation.py +++ b/tests/test_pcgr_annotation.py @@ -238,16 +238,10 @@ def test_missing_key_allow_missing_true_returns_record_unchanged(self): result = pcgr.annotate_record(variant, annotations, allow_missing=True) self.assertIsNotNone(result) - def test_missing_key_allow_missing_false_raises(self): - # NOTE: annotate_record's `assert key not in annotations` guard is tautological - # (it re-checks a condition already established by the enclosing `if`), so it never - # fires. Control falls through to `annotations[key].items()`, which raises KeyError - # instead of the AssertionError the guard's phrasing implies. Asserting the actual - # (KeyError) behavior here — the net effect (failing loudly on an unresolvable - # missing key) is preserved either way. + def test_missing_key_allow_missing_false_raises_assertion(self): variant = _make_variant(chrom='chr1', pos=100, ref='A', alt='T') annotations = {} - with self.assertRaises(KeyError): + with self.assertRaises(AssertionError): pcgr.annotate_record(variant, annotations, allow_missing=False) def test_key_uses_chrom_pos_ref_alt(self): From 3580ccd07184aa8dd27a3e1f3a2d1acfb2d6ab3d Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Tue, 7 Jul 2026 12:45:02 +1000 Subject: [PATCH 77/89] test: add coverage for estimate flags and disable_estimates forwarding - TestRunSomaticCommandArgs: assert --estimate_signatures absent from all run_somatic commands; assert disable_estimates=True suppresses --estimate_msi and --estimate_tmb - TestRunSomaticChunkArgMapping: assert run_somatic_chunk passes disable_estimates=True to every run_somatic call - Update CHANGELOG 0.3.2 with new test entries --- CHANGELOG.md | 2 + tests/test_pcgr_hypermutated.py | 104 ++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ddf05e2..8775add 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ - Test: `TestSelectPcgrVariants` — 8 integration tests covering tiered trimming, hotspot retention, PANEL retention, NONCODING-first drop order, and the `PCGR_MUTATION_HOTSPOT=.` regression - Test: `TestEntrySkipsPcgrOnOverflow` — 2 tests: entry() skips PCGR on `RuntimeError` from unresolvable overflow; entry() calls PCGR normally when within limit - Test: `TestSelectPcgrVariantsRaisesOnUnresolvableOverflow` — asserts `RuntimeError` when retained variants alone exceed `MAX_SOMATIC_VARIANTS` +- Test: `TestRunSomaticCommandArgs` — 2 tests: `--estimate_signatures` absent from all `run_somatic` commands; `disable_estimates=True` suppresses `--estimate_msi`/`--estimate_tmb` +- Test: `TestRunSomaticChunkArgMapping.test_disable_estimates_passed_to_run_somatic` — `run_somatic_chunk` passes `disable_estimates=True` to every `run_somatic` call ## 0.3.1 diff --git a/tests/test_pcgr_hypermutated.py b/tests/test_pcgr_hypermutated.py index 2082c4a..297c3f3 100644 --- a/tests/test_pcgr_hypermutated.py +++ b/tests/test_pcgr_hypermutated.py @@ -470,6 +470,110 @@ def fake_run_somatic(*args, **kwargs): self.assertEqual(kw.get('chunk_nbr'), 1, 'chunk_nbr was not forwarded correctly') + def test_disable_estimates_passed_to_run_somatic(self): + """run_somatic_chunk must pass disable_estimates=True to every run_somatic call. + + Chunked PCGR runs must not include --estimate_msi/--estimate_tmb per chunk — + those flags produce per-chunk partial estimates that are meaningless after merging. + Fixed in bolt 0.3.2 (umccr/sash#57 + disable_estimates wiring). + """ + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + vcf_fp = tmp_path / 'chunk.vcf' + _write_vcf(vcf_fp, [(10, f'PCGR_CSQ={_csq("intron_variant")}')]) + + captured = {} + + def fake_run_somatic(*args, **kwargs): + captured['kwargs'] = kwargs + return (None, None) + + with patch('bolt.common.pcgr.run_somatic', side_effect=fake_run_somatic), \ + patch('bolt.common.pcgr.merging_pcgr_files', + return_value=(tmp_path / 'out.tsv', tmp_path / 'out.vcf')): + pcgr.run_somatic_chunk( + [vcf_fp], + pcgr_data_dir=tmp_path / 'pcgr_data', + vep_dir=tmp_path / 'vep', + output_dir=tmp_path, + pcgr_output_dir=tmp_path / 'pcgr_output', + max_threads=4, + pcgr_conda='pcgr_env', + pcgrr_conda='pcgrr_env', + ) + + self.assertTrue( + captured['kwargs'].get('disable_estimates'), + 'run_somatic_chunk must forward disable_estimates=True — ' + 'per-chunk MSI/TMB estimates are meaningless after merge', + ) + + +class TestRunSomaticCommandArgs(unittest.TestCase): + """Verify run_somatic builds the correct PCGR command-line arguments.""" + + def test_estimate_signatures_absent_from_command(self): + """--estimate_signatures must not appear in the PCGR command (dropped in sash#57). + + --estimate_msi and --estimate_tmb must still be present for non-chunked runs. + """ + captured = {} + + def fake_execute(cmd, **kwargs): + captured['cmd'] = cmd + # create the output files run_somatic expects to find after pcgr runs + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / 'nosampleset.pcgr.grch38.snv_indel_ann.tsv.gz').touch() + (output_dir / 'nosampleset.pcgr.grch38.pass.vcf.gz').touch() + + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + output_dir = tmp_path / 'output' + + with patch('bolt.common.pcgr.util.execute_command', side_effect=fake_execute): + pcgr.run_somatic( + input_fp=tmp_path / 'input.vcf.gz', + pcgr_refdata_dir=tmp_path / 'refdata', + vep_dir=tmp_path / 'vep', + output_dir=output_dir, + ) + + self.assertIn('cmd', captured, 'execute_command was not called') + self.assertNotIn('--estimate_signatures', captured['cmd'], + '--estimate_signatures must be absent (dropped in sash#57)') + self.assertIn('--estimate_msi', captured['cmd'], + '--estimate_msi must still be present for non-chunked runs') + self.assertIn('--estimate_tmb', captured['cmd'], + '--estimate_tmb must still be present for non-chunked runs') + + def test_disable_estimates_suppresses_msi_tmb(self): + """disable_estimates=True must suppress --estimate_msi and --estimate_tmb.""" + captured = {} + + def fake_execute(cmd, **kwargs): + captured['cmd'] = cmd + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / 'nosampleset.pcgr.grch38.snv_indel_ann.tsv.gz').touch() + (output_dir / 'nosampleset.pcgr.grch38.pass.vcf.gz').touch() + + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + output_dir = tmp_path / 'output' + + with patch('bolt.common.pcgr.util.execute_command', side_effect=fake_execute): + pcgr.run_somatic( + input_fp=tmp_path / 'input.vcf.gz', + pcgr_refdata_dir=tmp_path / 'refdata', + vep_dir=tmp_path / 'vep', + output_dir=output_dir, + disable_estimates=True, + ) + + self.assertNotIn('--estimate_msi', captured['cmd'], + '--estimate_msi must be absent when disable_estimates=True') + self.assertNotIn('--estimate_tmb', captured['cmd'], + '--estimate_tmb must be absent when disable_estimates=True') + class TestCountVariantProcess(unittest.TestCase): """Verify count_variant_process counts and is_hypermutated flag (bolt #27). From 67ba9217b0d9d431eac44dabe77279fa4d73774b Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Tue, 7 Jul 2026 12:46:46 +1000 Subject: [PATCH 78/89] test(quick-n70): split hypermutated tests by source module Split test_pcgr_hypermutated.py into test_pcgr.py and test_smlv_somatic_report.py, matching bolt.common.pcgr and bolt.workflows.smlv_somatic.report; shared VCF fixtures moved to tests/helpers.py. --- tests/helpers.py | 46 ++ tests/test_pcgr.py | 368 +++++++++++++++ ...mutated.py => test_smlv_somatic_report.py} | 445 +----------------- 3 files changed, 440 insertions(+), 419 deletions(-) create mode 100644 tests/helpers.py create mode 100644 tests/test_pcgr.py rename tests/{test_pcgr_hypermutated.py => test_smlv_somatic_report.py} (51%) diff --git a/tests/helpers.py b/tests/helpers.py new file mode 100644 index 0000000..1ba472d --- /dev/null +++ b/tests/helpers.py @@ -0,0 +1,46 @@ +"""Shared VCF-construction fixtures used across test modules.""" +import pathlib +import tempfile + +import cyvcf2 + + +# Minimal CSQ: only tokens[1] (consequence) is read by get_impacts() +def _csq(consequence): + return f'A|{consequence}|.|.|.|.|.|.|.|.|.|.|.|.|.|.|.|.|.|.|.' + + +# Minimal VCF header with all INFO fields used by select_pcgr_variants +HEADER = ( + '##fileformat=VCFv4.2\n' + '##FILTER=\n' + '##INFO=\n' + '##INFO=\n' + '##INFO=\n' + '##INFO=\n' + '##INFO=\n' + '##INFO=\n' + '##INFO=\n' + '##INFO=\n' + '##contig=\n' + '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n' +) + + +def _write_vcf(path, variants): + with open(path, 'w') as fh: + fh.write(HEADER) + for pos, info in variants: + fh.write(f'chr1\t{pos}\t.\tA\tT\t.\tPASS\t{info}\n') + + +def _count_vcf(fp): + return sum(1 for _ in cyvcf2.VCF(str(fp))) + + +def _make_variant(info_str): + """Return a cyvcf2 Variant built from info_str using the test VCF header.""" + with tempfile.TemporaryDirectory() as tmp: + vcf_path = pathlib.Path(tmp) / 'test.vcf' + _write_vcf(vcf_path, [(100, info_str)]) + return list(cyvcf2.VCF(str(vcf_path)))[0] diff --git a/tests/test_pcgr.py b/tests/test_pcgr.py new file mode 100644 index 0000000..04759b4 --- /dev/null +++ b/tests/test_pcgr.py @@ -0,0 +1,368 @@ +"""Tests for bolt/common/pcgr.py — tier ordering, filter categorisation, chunking.""" +import pathlib +import tempfile +import unittest +from unittest.mock import patch + +import bolt.common.constants as constants +import bolt.common.pcgr as pcgr + +from tests.helpers import _csq, _count_vcf, _make_variant, _write_vcf + + +class TestTierOrdering(unittest.TestCase): + """Verify the PCGR_TIERS_FILTERING fix: values and priority order.""" + + def test_noncoding_filtered_before_tier1(self): + """N (NONCODING) entries must all precede '1' (TIER_1) entries in get_ordering().""" + ordering = pcgr.get_ordering() + tiers = [key[0] for key in ordering] + n_idx = [i for i, t in enumerate(tiers) if t == 'N'] + t1_idx = [i for i, t in enumerate(tiers) if t == '1'] + self.assertTrue(n_idx, 'No NONCODING (N) entries in get_ordering()') + self.assertTrue(t1_idx, 'No TIER_1 (1) entries in get_ordering()') + self.assertLess(max(n_idx), min(t1_idx), + 'All NONCODING entries must precede all TIER_1 entries') + + def test_no_long_form_tier_values(self): + """PCGR_TIERS_FILTERING must use short forms ('1'-'4', 'N'), not 'TIER_1' etc.""" + for v in constants.PCGR_TIERS_FILTERING: + self.assertNotIn('TIER_', v, + f"Found long-form tier value '{v}' — must be short form") + + def test_priority_order(self): + """Full ordering: N before 4 before 3 before 2 before 1.""" + expected = ('N', '4', '3', '2', '1') + self.assertEqual(constants.PCGR_TIERS_FILTERING, expected) + + +class TestGetImpacts(unittest.TestCase): + """Unit tests for pcgr.get_impacts() — CSQ string parsing.""" + + def test_single_consequence(self): + csq = _csq('intron_variant') + self.assertEqual(pcgr.get_impacts(csq), {'intron_variant'}) + + def test_multi_consequences_ampersand(self): + """A single CSQ entry with two consequences joined by & returns both.""" + csq = _csq('intron_variant&upstream_gene_variant') + self.assertEqual(pcgr.get_impacts(csq), {'intron_variant', 'upstream_gene_variant'}) + + def test_multiple_csq_entries_union(self): + """Comma-separated CSQ entries — returns the union of all consequences.""" + csq = f'{_csq("intron_variant")},{_csq("intergenic_variant")}' + self.assertEqual(pcgr.get_impacts(csq), {'intron_variant', 'intergenic_variant'}) + + +class TestDetermineFilter(unittest.TestCase): + """Unit tests for pcgr.determine_filter() — filter category determination.""" + + def _data(self, **overrides): + base = { + 'tier': None, + 'difficult': False, + 'giab_conf': False, + 'intergenic': None, + 'intronic': None, + 'downstream': None, + 'upstream': None, + 'impacts_other': None, + } + base.update(overrides) + return base + + def test_intergenic_difficult(self): + data = self._data(intergenic=True, difficult=True) + self.assertEqual(pcgr.determine_filter(data), ('intergenic', 'difficult')) + + def test_intergenic_no_region(self): + data = self._data(intergenic=True, difficult=False, giab_conf=False) + self.assertEqual(pcgr.determine_filter(data), ('intergenic', 'none')) + + def test_intergenic_giab_conf(self): + data = self._data(intergenic=True, giab_conf=True) + self.assertEqual(pcgr.determine_filter(data), ('intergenic', 'giab_conf')) + + def test_intronic_supersedes_intergenic(self): + """When both intergenic and intronic are present, intronic wins (higher priority).""" + data = self._data(intergenic=True, intronic=True, difficult=True) + self.assertEqual(pcgr.determine_filter(data), ('intronic', 'difficult')) + + def test_impacts_other_highest_priority(self): + """impacts_other is the last to be filtered — it wins over all other impacts.""" + data = self._data( + intergenic=True, intronic=True, downstream=True, + upstream=True, impacts_other=True, difficult=True, + ) + self.assertEqual(pcgr.determine_filter(data), ('impacts_other', 'difficult')) + + def test_no_impact_returns_false(self): + """A variant with no recognisable impact cannot be categorised.""" + data = self._data() # all impacts None + self.assertFalse(pcgr.determine_filter(data)) + + def test_giab_conf_region(self): + data = self._data(impacts_other=True, giab_conf=True) + self.assertEqual(pcgr.determine_filter(data), ('impacts_other', 'giab_conf')) + + +class TestGetVariantFilterData(unittest.TestCase): + """Unit tests for pcgr.get_variant_filter_data() — data extraction from VCF records.""" + + def test_tier_extracted(self): + info = f'PCGR_ACTIONABILITY_TIER=2;PCGR_CSQ={_csq("intron_variant")}' + data = pcgr.get_variant_filter_data(_make_variant(info)) + self.assertEqual(data['tier'], '2') + + def test_intergenic_impact(self): + info = f'PCGR_ACTIONABILITY_TIER=N;PCGR_CSQ={_csq("intergenic_variant")}' + data = pcgr.get_variant_filter_data(_make_variant(info)) + self.assertTrue(data['intergenic']) + self.assertFalse(data['intronic']) + self.assertFalse(data['downstream']) + self.assertFalse(data['upstream']) + self.assertFalse(data['impacts_other']) + + def test_intronic_impact(self): + info = f'PCGR_ACTIONABILITY_TIER=1;PCGR_CSQ={_csq("intron_variant")}' + data = pcgr.get_variant_filter_data(_make_variant(info)) + self.assertTrue(data['intronic']) + self.assertFalse(data['intergenic']) + + def test_giab_conf_overrides_difficult(self): + """GIAB_CONF flag must clear the difficult flag even when DIFFICULT_* is also present.""" + info = f'GIAB_CONF;DIFFICULT_segdup;PCGR_ACTIONABILITY_TIER=1;PCGR_CSQ={_csq("intron_variant")}' + data = pcgr.get_variant_filter_data(_make_variant(info)) + self.assertTrue(data['giab_conf']) + self.assertFalse(data['difficult']) + + def test_difficult_without_giab(self): + info = f'DIFFICULT_segdup;PCGR_ACTIONABILITY_TIER=1;PCGR_CSQ={_csq("intron_variant")}' + data = pcgr.get_variant_filter_data(_make_variant(info)) + self.assertTrue(data['difficult']) + self.assertFalse(data['giab_conf']) + + +class TestSplitVcf(unittest.TestCase): + """Tests for pcgr.split_vcf() — chunking the annotation path for large VCFs. + + split_vcf() is the annotate-path strategy for hypermutated samples: it divides + a VCF into ≤MAX_SOMATIC_VARIANTS chunks so each chunk can be run through PCGR + independently. Tested 2026-05-13 with a synthetic 550k VCF: 550k → 450k + 100k. + """ + + def test_chunks_above_limit(self): + """VCF exceeding the limit is split into correctly-sized chunks.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + vcf_fp = tmp_path / 'input.vcf' + # 25 variants, limit=10 → expect 3 chunks (10, 10, 5) + v = [(i * 10, f'PCGR_CSQ={_csq("intron_variant")}') for i in range(1, 26)] + _write_vcf(vcf_fp, v) + + with patch('bolt.common.constants.MAX_SOMATIC_VARIANTS', 10): + chunks = pcgr.split_vcf(vcf_fp, tmp_path) + + self.assertEqual(len(chunks), 3) + counts = [_count_vcf(c) for c in chunks] + self.assertLessEqual(max(counts), 10) + self.assertEqual(sum(counts), 25) + + def test_no_chunking_within_limit(self): + """VCF within the limit produces a single chunk containing all variants.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + vcf_fp = tmp_path / 'input.vcf' + v = [(i * 10, f'PCGR_CSQ={_csq("intron_variant")}') for i in range(1, 6)] + _write_vcf(vcf_fp, v) + + with patch('bolt.common.constants.MAX_SOMATIC_VARIANTS', 10): + chunks = pcgr.split_vcf(vcf_fp, tmp_path) + + self.assertEqual(len(chunks), 1) + self.assertEqual(_count_vcf(chunks[0]), 5) + + def test_chunks_are_gzipped(self): + """Chunk files must be .vcf.gz — plain .vcf chunks violate CLAUDE.md and waste disk.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + vcf_fp = tmp_path / 'input.vcf' + v = [(i * 10, f'PCGR_CSQ={_csq("intron_variant")}') for i in range(1, 26)] + _write_vcf(vcf_fp, v) + + with patch('bolt.common.constants.MAX_SOMATIC_VARIANTS', 10): + chunks = pcgr.split_vcf(vcf_fp, tmp_path) + + for chunk in chunks: + self.assertTrue(str(chunk).endswith('.vcf.gz'), + f'Expected .vcf.gz chunk, got: {chunk.name}') + + def test_chunks_are_tabix_indexed(self): + """Each .vcf.gz chunk must have a .tbi index — PCGR v2.2.5 requires it.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + vcf_fp = tmp_path / 'input.vcf' + v = [(i * 10, f'PCGR_CSQ={_csq("intron_variant")}') for i in range(1, 26)] + _write_vcf(vcf_fp, v) + + with patch('bolt.common.constants.MAX_SOMATIC_VARIANTS', 10): + chunks = pcgr.split_vcf(vcf_fp, tmp_path) + + for chunk in chunks: + tbi = pathlib.Path(str(chunk) + '.tbi') + self.assertTrue(tbi.exists(), f'Missing tabix index for {chunk.name}') + + +class TestRunSomaticChunkArgMapping(unittest.TestCase): + """Regression test: run_somatic_chunk must forward args as keywords to run_somatic. + + Before the fix, run_somatic_chunk called run_somatic positionally (6 args), + skipping pcgr_threads. This caused pcgr_conda ('pcgr') to land in the + pcgr_threads slot → ValueError: invalid literal for int() with base 10: 'pcgr'. + """ + + def test_pcgr_conda_not_shifted_into_pcgr_threads(self): + """pcgr_conda must reach run_somatic as pcgr_conda, not as pcgr_threads.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + vcf_fp = tmp_path / 'chunk.vcf' + _write_vcf(vcf_fp, [(10, f'PCGR_CSQ={_csq("intron_variant")}')]) + + captured = {} + + def fake_run_somatic(*args, **kwargs): + captured['args'] = args + captured['kwargs'] = kwargs + return (None, None) + + with patch('bolt.common.pcgr.run_somatic', side_effect=fake_run_somatic), \ + patch('bolt.common.pcgr.merging_pcgr_files', + return_value=(tmp_path / 'out.vcf', tmp_path / 'out.tsv')): + pcgr.run_somatic_chunk( + [vcf_fp], + pcgr_data_dir=tmp_path / 'pcgr_data', + vep_dir=tmp_path / 'vep', + output_dir=tmp_path, + pcgr_output_dir=tmp_path / 'pcgr_output', + max_threads=4, + pcgr_conda='pcgr_env', + pcgrr_conda='pcgrr_env', + ) + + kw = captured['kwargs'] + self.assertEqual(kw.get('pcgr_conda'), 'pcgr_env', + 'pcgr_conda was not forwarded — likely shifted into pcgr_threads') + self.assertEqual(kw.get('pcgrr_conda'), 'pcgrr_env', + 'pcgrr_conda was not forwarded correctly') + self.assertEqual(kw.get('threads'), 4, + 'threads (max_threads) was not forwarded correctly') + self.assertEqual(kw.get('chunk_nbr'), 1, + 'chunk_nbr was not forwarded correctly') + + def test_disable_estimates_passed_to_run_somatic(self): + """run_somatic_chunk must pass disable_estimates=True to every run_somatic call. + + Chunked PCGR runs must not include --estimate_msi/--estimate_tmb per chunk — + those flags produce per-chunk partial estimates that are meaningless after merging. + Fixed in bolt 0.3.2 (umccr/sash#57 + disable_estimates wiring). + """ + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + vcf_fp = tmp_path / 'chunk.vcf' + _write_vcf(vcf_fp, [(10, f'PCGR_CSQ={_csq("intron_variant")}')]) + + captured = {} + + def fake_run_somatic(*args, **kwargs): + captured['kwargs'] = kwargs + return (None, None) + + with patch('bolt.common.pcgr.run_somatic', side_effect=fake_run_somatic), \ + patch('bolt.common.pcgr.merging_pcgr_files', + return_value=(tmp_path / 'out.tsv', tmp_path / 'out.vcf')): + pcgr.run_somatic_chunk( + [vcf_fp], + pcgr_data_dir=tmp_path / 'pcgr_data', + vep_dir=tmp_path / 'vep', + output_dir=tmp_path, + pcgr_output_dir=tmp_path / 'pcgr_output', + max_threads=4, + pcgr_conda='pcgr_env', + pcgrr_conda='pcgrr_env', + ) + + self.assertTrue( + captured['kwargs'].get('disable_estimates'), + 'run_somatic_chunk must forward disable_estimates=True — ' + 'per-chunk MSI/TMB estimates are meaningless after merge', + ) + + +class TestRunSomaticCommandArgs(unittest.TestCase): + """Verify run_somatic builds the correct PCGR command-line arguments.""" + + def test_estimate_signatures_absent_from_command(self): + """--estimate_signatures must not appear in the PCGR command (dropped in sash#57). + + --estimate_msi and --estimate_tmb must still be present for non-chunked runs. + """ + captured = {} + + def fake_execute(cmd, **kwargs): + captured['cmd'] = cmd + # create the output files run_somatic expects to find after pcgr runs + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / 'nosampleset.pcgr.grch38.snv_indel_ann.tsv.gz').touch() + (output_dir / 'nosampleset.pcgr.grch38.pass.vcf.gz').touch() + + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + output_dir = tmp_path / 'output' + + with patch('bolt.common.pcgr.util.execute_command', side_effect=fake_execute): + pcgr.run_somatic( + input_fp=tmp_path / 'input.vcf.gz', + pcgr_refdata_dir=tmp_path / 'refdata', + vep_dir=tmp_path / 'vep', + output_dir=output_dir, + ) + + self.assertIn('cmd', captured, 'execute_command was not called') + self.assertNotIn('--estimate_signatures', captured['cmd'], + '--estimate_signatures must be absent (dropped in sash#57)') + self.assertIn('--estimate_msi', captured['cmd'], + '--estimate_msi must still be present for non-chunked runs') + self.assertIn('--estimate_tmb', captured['cmd'], + '--estimate_tmb must still be present for non-chunked runs') + + def test_disable_estimates_suppresses_msi_tmb(self): + """disable_estimates=True must suppress --estimate_msi and --estimate_tmb.""" + captured = {} + + def fake_execute(cmd, **kwargs): + captured['cmd'] = cmd + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / 'nosampleset.pcgr.grch38.snv_indel_ann.tsv.gz').touch() + (output_dir / 'nosampleset.pcgr.grch38.pass.vcf.gz').touch() + + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + output_dir = tmp_path / 'output' + + with patch('bolt.common.pcgr.util.execute_command', side_effect=fake_execute): + pcgr.run_somatic( + input_fp=tmp_path / 'input.vcf.gz', + pcgr_refdata_dir=tmp_path / 'refdata', + vep_dir=tmp_path / 'vep', + output_dir=output_dir, + disable_estimates=True, + ) + + self.assertNotIn('--estimate_msi', captured['cmd'], + '--estimate_msi must be absent when disable_estimates=True') + self.assertNotIn('--estimate_tmb', captured['cmd'], + '--estimate_tmb must be absent when disable_estimates=True') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_pcgr_hypermutated.py b/tests/test_smlv_somatic_report.py similarity index 51% rename from tests/test_pcgr_hypermutated.py rename to tests/test_smlv_somatic_report.py index 297c3f3..7953808 100644 --- a/tests/test_pcgr_hypermutated.py +++ b/tests/test_smlv_somatic_report.py @@ -1,4 +1,4 @@ -"""Tests for hypermutated sample handling — tier ordering fix and variant trimming.""" +"""Tests for bolt/workflows/smlv_somatic/report.py — hypermutated variant trimming.""" import pathlib import shutil import tempfile @@ -14,72 +14,7 @@ import bolt.util as util import bolt.workflows.smlv_somatic.report as report_mod - -# Minimal CSQ: only tokens[1] (consequence) is read by get_impacts() -def _csq(consequence): - return f'A|{consequence}|.|.|.|.|.|.|.|.|.|.|.|.|.|.|.|.|.|.|.' - - -# Minimal VCF header with all INFO fields used by select_pcgr_variants -HEADER = ( - '##fileformat=VCFv4.2\n' - '##FILTER=\n' - '##INFO=\n' - '##INFO=\n' - '##INFO=\n' - '##INFO=\n' - '##INFO=\n' - '##INFO=\n' - '##INFO=\n' - '##INFO=\n' - '##contig=\n' - '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n' -) - - -def _write_vcf(path, variants): - with open(path, 'w') as fh: - fh.write(HEADER) - for pos, info in variants: - fh.write(f'chr1\t{pos}\t.\tA\tT\t.\tPASS\t{info}\n') - - -def _count_vcf(fp): - return sum(1 for _ in cyvcf2.VCF(str(fp))) - - -def _make_variant(info_str): - """Return a cyvcf2 Variant built from info_str using the test VCF header.""" - with tempfile.TemporaryDirectory() as tmp: - vcf_path = pathlib.Path(tmp) / 'test.vcf' - _write_vcf(vcf_path, [(100, info_str)]) - return list(cyvcf2.VCF(str(vcf_path)))[0] - - -class TestTierOrdering(unittest.TestCase): - """Verify the PCGR_TIERS_FILTERING fix: values and priority order.""" - - def test_noncoding_filtered_before_tier1(self): - """N (NONCODING) entries must all precede '1' (TIER_1) entries in get_ordering().""" - ordering = pcgr.get_ordering() - tiers = [key[0] for key in ordering] - n_idx = [i for i, t in enumerate(tiers) if t == 'N'] - t1_idx = [i for i, t in enumerate(tiers) if t == '1'] - self.assertTrue(n_idx, 'No NONCODING (N) entries in get_ordering()') - self.assertTrue(t1_idx, 'No TIER_1 (1) entries in get_ordering()') - self.assertLess(max(n_idx), min(t1_idx), - 'All NONCODING entries must precede all TIER_1 entries') - - def test_no_long_form_tier_values(self): - """PCGR_TIERS_FILTERING must use short forms ('1'-'4', 'N'), not 'TIER_1' etc.""" - for v in constants.PCGR_TIERS_FILTERING: - self.assertNotIn('TIER_', v, - f"Found long-form tier value '{v}' — must be short form") - - def test_priority_order(self): - """Full ordering: N before 4 before 3 before 2 before 1.""" - expected = ('N', '4', '3', '2', '1') - self.assertEqual(constants.PCGR_TIERS_FILTERING, expected) +from tests.helpers import HEADER, _csq, _write_vcf class TestSelectPcgrVariants(unittest.TestCase): @@ -107,7 +42,7 @@ def fake_execute(cmd, **_): out_fp = report_mod.select_pcgr_variants( vcf_fp, cancer_genes, 'TUMOR', pathlib.Path(tmp) ) - return _count_vcf(out_fp) + return sum(1 for _ in cyvcf2.VCF(str(out_fp))) def test_output_within_limit(self): """Output must never exceed MAX_SOMATIC_VARIANTS.""" @@ -247,332 +182,35 @@ def fake_execute(cmd, **_): self.assertEqual(len(dropped), 2, 'Expected 2 NONCODING variants marked with PCGR_count_limit') -class TestGetImpacts(unittest.TestCase): - """Unit tests for pcgr.get_impacts() — CSQ string parsing.""" - - def test_single_consequence(self): - csq = _csq('intron_variant') - self.assertEqual(pcgr.get_impacts(csq), {'intron_variant'}) - - def test_multi_consequences_ampersand(self): - """A single CSQ entry with two consequences joined by & returns both.""" - csq = _csq('intron_variant&upstream_gene_variant') - self.assertEqual(pcgr.get_impacts(csq), {'intron_variant', 'upstream_gene_variant'}) - - def test_multiple_csq_entries_union(self): - """Comma-separated CSQ entries — returns the union of all consequences.""" - csq = f'{_csq("intron_variant")},{_csq("intergenic_variant")}' - self.assertEqual(pcgr.get_impacts(csq), {'intron_variant', 'intergenic_variant'}) - - -class TestDetermineFilter(unittest.TestCase): - """Unit tests for pcgr.determine_filter() — filter category determination.""" - - def _data(self, **overrides): - base = { - 'tier': None, - 'difficult': False, - 'giab_conf': False, - 'intergenic': None, - 'intronic': None, - 'downstream': None, - 'upstream': None, - 'impacts_other': None, - } - base.update(overrides) - return base - - def test_intergenic_difficult(self): - data = self._data(intergenic=True, difficult=True) - self.assertEqual(pcgr.determine_filter(data), ('intergenic', 'difficult')) - - def test_intergenic_no_region(self): - data = self._data(intergenic=True, difficult=False, giab_conf=False) - self.assertEqual(pcgr.determine_filter(data), ('intergenic', 'none')) - - def test_intergenic_giab_conf(self): - data = self._data(intergenic=True, giab_conf=True) - self.assertEqual(pcgr.determine_filter(data), ('intergenic', 'giab_conf')) - - def test_intronic_supersedes_intergenic(self): - """When both intergenic and intronic are present, intronic wins (higher priority).""" - data = self._data(intergenic=True, intronic=True, difficult=True) - self.assertEqual(pcgr.determine_filter(data), ('intronic', 'difficult')) - - def test_impacts_other_highest_priority(self): - """impacts_other is the last to be filtered — it wins over all other impacts.""" - data = self._data( - intergenic=True, intronic=True, downstream=True, - upstream=True, impacts_other=True, difficult=True, - ) - self.assertEqual(pcgr.determine_filter(data), ('impacts_other', 'difficult')) - - def test_no_impact_returns_false(self): - """A variant with no recognisable impact cannot be categorised.""" - data = self._data() # all impacts None - self.assertFalse(pcgr.determine_filter(data)) - - def test_giab_conf_region(self): - data = self._data(impacts_other=True, giab_conf=True) - self.assertEqual(pcgr.determine_filter(data), ('impacts_other', 'giab_conf')) - - -class TestGetVariantFilterData(unittest.TestCase): - """Unit tests for pcgr.get_variant_filter_data() — data extraction from VCF records.""" - - def test_tier_extracted(self): - info = f'PCGR_ACTIONABILITY_TIER=2;PCGR_CSQ={_csq("intron_variant")}' - data = pcgr.get_variant_filter_data(_make_variant(info)) - self.assertEqual(data['tier'], '2') - - def test_intergenic_impact(self): - info = f'PCGR_ACTIONABILITY_TIER=N;PCGR_CSQ={_csq("intergenic_variant")}' - data = pcgr.get_variant_filter_data(_make_variant(info)) - self.assertTrue(data['intergenic']) - self.assertFalse(data['intronic']) - self.assertFalse(data['downstream']) - self.assertFalse(data['upstream']) - self.assertFalse(data['impacts_other']) - - def test_intronic_impact(self): - info = f'PCGR_ACTIONABILITY_TIER=1;PCGR_CSQ={_csq("intron_variant")}' - data = pcgr.get_variant_filter_data(_make_variant(info)) - self.assertTrue(data['intronic']) - self.assertFalse(data['intergenic']) - - def test_giab_conf_overrides_difficult(self): - """GIAB_CONF flag must clear the difficult flag even when DIFFICULT_* is also present.""" - info = f'GIAB_CONF;DIFFICULT_segdup;PCGR_ACTIONABILITY_TIER=1;PCGR_CSQ={_csq("intron_variant")}' - data = pcgr.get_variant_filter_data(_make_variant(info)) - self.assertTrue(data['giab_conf']) - self.assertFalse(data['difficult']) - - def test_difficult_without_giab(self): - info = f'DIFFICULT_segdup;PCGR_ACTIONABILITY_TIER=1;PCGR_CSQ={_csq("intron_variant")}' - data = pcgr.get_variant_filter_data(_make_variant(info)) - self.assertTrue(data['difficult']) - self.assertFalse(data['giab_conf']) - - -class TestSplitVcf(unittest.TestCase): - """Tests for pcgr.split_vcf() — chunking the annotation path for large VCFs. - - split_vcf() is the annotate-path strategy for hypermutated samples: it divides - a VCF into ≤MAX_SOMATIC_VARIANTS chunks so each chunk can be run through PCGR - independently. Tested 2026-05-13 with a synthetic 550k VCF: 550k → 450k + 100k. - """ - - def test_chunks_above_limit(self): - """VCF exceeding the limit is split into correctly-sized chunks.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_path = pathlib.Path(tmp) - vcf_fp = tmp_path / 'input.vcf' - # 25 variants, limit=10 → expect 3 chunks (10, 10, 5) - v = [(i * 10, f'PCGR_CSQ={_csq("intron_variant")}') for i in range(1, 26)] - _write_vcf(vcf_fp, v) - - with patch('bolt.common.constants.MAX_SOMATIC_VARIANTS', 10): - chunks = pcgr.split_vcf(vcf_fp, tmp_path) - - self.assertEqual(len(chunks), 3) - counts = [_count_vcf(c) for c in chunks] - self.assertLessEqual(max(counts), 10) - self.assertEqual(sum(counts), 25) - - def test_no_chunking_within_limit(self): - """VCF within the limit produces a single chunk containing all variants.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_path = pathlib.Path(tmp) - vcf_fp = tmp_path / 'input.vcf' - v = [(i * 10, f'PCGR_CSQ={_csq("intron_variant")}') for i in range(1, 6)] - _write_vcf(vcf_fp, v) - - with patch('bolt.common.constants.MAX_SOMATIC_VARIANTS', 10): - chunks = pcgr.split_vcf(vcf_fp, tmp_path) - - self.assertEqual(len(chunks), 1) - self.assertEqual(_count_vcf(chunks[0]), 5) - - def test_chunks_are_gzipped(self): - """Chunk files must be .vcf.gz — plain .vcf chunks violate CLAUDE.md and waste disk.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_path = pathlib.Path(tmp) - vcf_fp = tmp_path / 'input.vcf' - v = [(i * 10, f'PCGR_CSQ={_csq("intron_variant")}') for i in range(1, 26)] - _write_vcf(vcf_fp, v) - - with patch('bolt.common.constants.MAX_SOMATIC_VARIANTS', 10): - chunks = pcgr.split_vcf(vcf_fp, tmp_path) +class TestSelectPcgrVariantsRaisesOnUnresolvableOverflow(unittest.TestCase): + """select_pcgr_variants raises RuntimeError when all variants are retained (hotspots).""" - for chunk in chunks: - self.assertTrue(str(chunk).endswith('.vcf.gz'), - f'Expected .vcf.gz chunk, got: {chunk.name}') + def _fake_execute(self, vcf_fp): + def _run(cmd, **_): + import re + m = re.search(r'--output\s+(\S+)', cmd) + if m and 'bcftools annotate' in cmd: + shutil.copy(str(vcf_fp), m.group(1)) + else: + util.execute_command(cmd) + return _run - def test_chunks_are_tabix_indexed(self): - """Each .vcf.gz chunk must have a .tbi index — PCGR v2.2.5 requires it.""" + def test_raises_when_all_variants_are_hotspots(self): + """All SAGE_HOTSPOT variants are RETAIN_FIELDS — tiered filtering cannot drop any; RuntimeError expected.""" + # RETAIN_FIELDS_FILTERING includes SAGE_HOTSPOT — use that flag, not HMF_HOTSPOT + HOTSPOT_INFO = f'SAGE_HOTSPOT;PCGR_ACTIONABILITY_TIER=1;PCGR_CSQ={_csq("intron_variant")}' with tempfile.TemporaryDirectory() as tmp: tmp_path = pathlib.Path(tmp) vcf_fp = tmp_path / 'input.vcf' - v = [(i * 10, f'PCGR_CSQ={_csq("intron_variant")}') for i in range(1, 26)] - _write_vcf(vcf_fp, v) - - with patch('bolt.common.constants.MAX_SOMATIC_VARIANTS', 10): - chunks = pcgr.split_vcf(vcf_fp, tmp_path) - - for chunk in chunks: - tbi = pathlib.Path(str(chunk) + '.tbi') - self.assertTrue(tbi.exists(), f'Missing tabix index for {chunk.name}') - - -class TestRunSomaticChunkArgMapping(unittest.TestCase): - """Regression test: run_somatic_chunk must forward args as keywords to run_somatic. - - Before the fix, run_somatic_chunk called run_somatic positionally (6 args), - skipping pcgr_threads. This caused pcgr_conda ('pcgr') to land in the - pcgr_threads slot → ValueError: invalid literal for int() with base 10: 'pcgr'. - """ - - def test_pcgr_conda_not_shifted_into_pcgr_threads(self): - """pcgr_conda must reach run_somatic as pcgr_conda, not as pcgr_threads.""" - with tempfile.TemporaryDirectory() as tmp: - tmp_path = pathlib.Path(tmp) - vcf_fp = tmp_path / 'chunk.vcf' - _write_vcf(vcf_fp, [(10, f'PCGR_CSQ={_csq("intron_variant")}')]) - - captured = {} - - def fake_run_somatic(*args, **kwargs): - captured['args'] = args - captured['kwargs'] = kwargs - return (None, None) - - with patch('bolt.common.pcgr.run_somatic', side_effect=fake_run_somatic), \ - patch('bolt.common.pcgr.merging_pcgr_files', - return_value=(tmp_path / 'out.vcf', tmp_path / 'out.tsv')): - pcgr.run_somatic_chunk( - [vcf_fp], - pcgr_data_dir=tmp_path / 'pcgr_data', - vep_dir=tmp_path / 'vep', - output_dir=tmp_path, - pcgr_output_dir=tmp_path / 'pcgr_output', - max_threads=4, - pcgr_conda='pcgr_env', - pcgrr_conda='pcgrr_env', - ) - - kw = captured['kwargs'] - self.assertEqual(kw.get('pcgr_conda'), 'pcgr_env', - 'pcgr_conda was not forwarded — likely shifted into pcgr_threads') - self.assertEqual(kw.get('pcgrr_conda'), 'pcgrr_env', - 'pcgrr_conda was not forwarded correctly') - self.assertEqual(kw.get('threads'), 4, - 'threads (max_threads) was not forwarded correctly') - self.assertEqual(kw.get('chunk_nbr'), 1, - 'chunk_nbr was not forwarded correctly') - - def test_disable_estimates_passed_to_run_somatic(self): - """run_somatic_chunk must pass disable_estimates=True to every run_somatic call. - - Chunked PCGR runs must not include --estimate_msi/--estimate_tmb per chunk — - those flags produce per-chunk partial estimates that are meaningless after merging. - Fixed in bolt 0.3.2 (umccr/sash#57 + disable_estimates wiring). - """ - with tempfile.TemporaryDirectory() as tmp: - tmp_path = pathlib.Path(tmp) - vcf_fp = tmp_path / 'chunk.vcf' - _write_vcf(vcf_fp, [(10, f'PCGR_CSQ={_csq("intron_variant")}')]) - - captured = {} - - def fake_run_somatic(*args, **kwargs): - captured['kwargs'] = kwargs - return (None, None) - - with patch('bolt.common.pcgr.run_somatic', side_effect=fake_run_somatic), \ - patch('bolt.common.pcgr.merging_pcgr_files', - return_value=(tmp_path / 'out.tsv', tmp_path / 'out.vcf')): - pcgr.run_somatic_chunk( - [vcf_fp], - pcgr_data_dir=tmp_path / 'pcgr_data', - vep_dir=tmp_path / 'vep', - output_dir=tmp_path, - pcgr_output_dir=tmp_path / 'pcgr_output', - max_threads=4, - pcgr_conda='pcgr_env', - pcgrr_conda='pcgrr_env', - ) - - self.assertTrue( - captured['kwargs'].get('disable_estimates'), - 'run_somatic_chunk must forward disable_estimates=True — ' - 'per-chunk MSI/TMB estimates are meaningless after merge', - ) - - -class TestRunSomaticCommandArgs(unittest.TestCase): - """Verify run_somatic builds the correct PCGR command-line arguments.""" - - def test_estimate_signatures_absent_from_command(self): - """--estimate_signatures must not appear in the PCGR command (dropped in sash#57). - - --estimate_msi and --estimate_tmb must still be present for non-chunked runs. - """ - captured = {} - - def fake_execute(cmd, **kwargs): - captured['cmd'] = cmd - # create the output files run_somatic expects to find after pcgr runs - output_dir.mkdir(parents=True, exist_ok=True) - (output_dir / 'nosampleset.pcgr.grch38.snv_indel_ann.tsv.gz').touch() - (output_dir / 'nosampleset.pcgr.grch38.pass.vcf.gz').touch() - - with tempfile.TemporaryDirectory() as tmp: - tmp_path = pathlib.Path(tmp) - output_dir = tmp_path / 'output' - - with patch('bolt.common.pcgr.util.execute_command', side_effect=fake_execute): - pcgr.run_somatic( - input_fp=tmp_path / 'input.vcf.gz', - pcgr_refdata_dir=tmp_path / 'refdata', - vep_dir=tmp_path / 'vep', - output_dir=output_dir, - ) - - self.assertIn('cmd', captured, 'execute_command was not called') - self.assertNotIn('--estimate_signatures', captured['cmd'], - '--estimate_signatures must be absent (dropped in sash#57)') - self.assertIn('--estimate_msi', captured['cmd'], - '--estimate_msi must still be present for non-chunked runs') - self.assertIn('--estimate_tmb', captured['cmd'], - '--estimate_tmb must still be present for non-chunked runs') - - def test_disable_estimates_suppresses_msi_tmb(self): - """disable_estimates=True must suppress --estimate_msi and --estimate_tmb.""" - captured = {} - - def fake_execute(cmd, **kwargs): - captured['cmd'] = cmd - output_dir.mkdir(parents=True, exist_ok=True) - (output_dir / 'nosampleset.pcgr.grch38.snv_indel_ann.tsv.gz').touch() - (output_dir / 'nosampleset.pcgr.grch38.pass.vcf.gz').touch() - - with tempfile.TemporaryDirectory() as tmp: - tmp_path = pathlib.Path(tmp) - output_dir = tmp_path / 'output' - - with patch('bolt.common.pcgr.util.execute_command', side_effect=fake_execute): - pcgr.run_somatic( - input_fp=tmp_path / 'input.vcf.gz', - pcgr_refdata_dir=tmp_path / 'refdata', - vep_dir=tmp_path / 'vep', - output_dir=output_dir, - disable_estimates=True, - ) + variants = [(i * 10, HOTSPOT_INFO) for i in range(1, 6)] + _write_vcf(vcf_fp, variants) + cancer_genes = tmp_path / 'genes.bed' + cancer_genes.write_text('chr1\t1\t9999999\n') - self.assertNotIn('--estimate_msi', captured['cmd'], - '--estimate_msi must be absent when disable_estimates=True') - self.assertNotIn('--estimate_tmb', captured['cmd'], - '--estimate_tmb must be absent when disable_estimates=True') + with patch('bolt.common.constants.MAX_SOMATIC_VARIANTS', 3), \ + patch('bolt.util.execute_command', side_effect=self._fake_execute(vcf_fp)): + with self.assertRaises(RuntimeError): + report_mod.select_pcgr_variants(vcf_fp, cancer_genes, 'TUMOR', tmp_path) class TestCountVariantProcess(unittest.TestCase): @@ -670,37 +308,6 @@ def test_annotation_filter_excluded_from_annotated_count(self): self.assertEqual(counts['dragen'], 2) -class TestSelectPcgrVariantsRaisesOnUnresolvableOverflow(unittest.TestCase): - """select_pcgr_variants raises RuntimeError when all variants are retained (hotspots).""" - - def _fake_execute(self, vcf_fp): - def _run(cmd, **_): - import re - m = re.search(r'--output\s+(\S+)', cmd) - if m and 'bcftools annotate' in cmd: - shutil.copy(str(vcf_fp), m.group(1)) - else: - util.execute_command(cmd) - return _run - - def test_raises_when_all_variants_are_hotspots(self): - """All SAGE_HOTSPOT variants are RETAIN_FIELDS — tiered filtering cannot drop any; RuntimeError expected.""" - # RETAIN_FIELDS_FILTERING includes SAGE_HOTSPOT — use that flag, not HMF_HOTSPOT - HOTSPOT_INFO = f'SAGE_HOTSPOT;PCGR_ACTIONABILITY_TIER=1;PCGR_CSQ={_csq("intron_variant")}' - with tempfile.TemporaryDirectory() as tmp: - tmp_path = pathlib.Path(tmp) - vcf_fp = tmp_path / 'input.vcf' - variants = [(i * 10, HOTSPOT_INFO) for i in range(1, 6)] - _write_vcf(vcf_fp, variants) - cancer_genes = tmp_path / 'genes.bed' - cancer_genes.write_text('chr1\t1\t9999999\n') - - with patch('bolt.common.constants.MAX_SOMATIC_VARIANTS', 3), \ - patch('bolt.util.execute_command', side_effect=self._fake_execute(vcf_fp)): - with self.assertRaises(RuntimeError): - report_mod.select_pcgr_variants(vcf_fp, cancer_genes, 'TUMOR', tmp_path) - - _PASS_COUNTS = {'pass': {'snps': 0, 'indels': 0, 'others': 0, 'total': 0}} From 331ee1e0b811b53433b1b2389cfca1ad89a31b91 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Tue, 7 Jul 2026 13:42:05 +1000 Subject: [PATCH 79/89] fix: address Copilot review findings on PR #33 - parse_genomic_change: guard against double 'chr' prefix if GENOMIC_CHANGE already includes it, matching the defensive check already used in get_annotation_entry_tsv's fallback path - setup_logging: force=True on basicConfig so handler setup is deterministic regardless of prior logging configuration in-process - annotate.py: fix stale comment listing transferred PCGR annotations to match transfer_annotations_somatic's actual info_field_map --- bolt/common/pcgr.py | 4 +++- bolt/logging_config.py | 3 ++- bolt/workflows/smlv_somatic/annotate.py | 7 +++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/bolt/common/pcgr.py b/bolt/common/pcgr.py index 7727cf8..019be83 100644 --- a/bolt/common/pcgr.py +++ b/bolt/common/pcgr.py @@ -418,7 +418,9 @@ def parse_genomic_change(genomic_change): match = re.match(pattern, genomic_change) if not match: raise ValueError(f"Format not recognized: {genomic_change}") - chrom = f"chr{match.group('chrom')}" + chrom = match.group('chrom') + if not chrom.startswith('chr'): + chrom = f'chr{chrom}' pos = int(match.group('pos')) ref = match.group('ref') alt = match.group('alt') diff --git a/bolt/logging_config.py b/bolt/logging_config.py index 0ddda2b..66b80ee 100644 --- a/bolt/logging_config.py +++ b/bolt/logging_config.py @@ -28,7 +28,8 @@ def setup_logging(output_dir, script_name): logging.basicConfig( level=logging.DEBUG, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - handlers=[file_handler, console_handler] + handlers=[file_handler, console_handler], + force=True, ) logger = logging.getLogger(__name__) logger.info("Logging setup complete") \ No newline at end of file diff --git a/bolt/workflows/smlv_somatic/annotate.py b/bolt/workflows/smlv_somatic/annotate.py index 1e4e434..78c07df 100644 --- a/bolt/workflows/smlv_somatic/annotate.py +++ b/bolt/workflows/smlv_somatic/annotate.py @@ -80,12 +80,11 @@ def entry(ctx, **kwargs): # - Set tumor and normal AF and DP in INFO for PCGR and remove all other annotations # - Run PCGR on minimal VCF (pcgr_prep_fp) # - Transfer selected PCGR annotations to unfiltered VCF (selected_fp) - # - PCGR ACMG TIER [INFO/PCGR_TIER] - # - VEP consequence [INFO/PCR_CSQ] + # - PCGR actionability tier [INFO/PCGR_ACTIONABILITY_TIER] + # - VEP consequence [INFO/PCGR_CSQ] # - Known mutation hotspot [INFO/PCGR_MUTATION_HOTSPOT] - # - ClinVar clinical significant [INFO/PCGR_CLNSIG] + # - ClinVar classification [INFO/PCGR_CLINVAR_CLASSIFICATION] # - Hits in TCGA [INFO/PCGR_TCGA_PANCANCER_COUNT] - # - Hits in PCAWG [INFO/PCGR_ICGC_PCAWG_COUNT] # Prepare VCF for PCGR annotation pcgr_prep_fp = pcgr.prepare_vcf_somatic( From fb51f0e34cfdb5c31140ef917091fb8ecc48bed8 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Tue, 7 Jul 2026 13:42:30 +1000 Subject: [PATCH 80/89] docs(quick-n70): fix stale test-file references after pcgr test split TESTING.md and test docstrings still pointed at the deleted tests/test_pcgr_hypermutated.py; update to tests/test_pcgr.py and tests/test_smlv_somatic_report.py. Also drop an unused HEADER import left over from the split. --- TESTING.md | 6 +++--- tests/test_pcgr_annotation.py | 2 +- tests/test_smlv_somatic_report.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/TESTING.md b/TESTING.md index 791eb9e..e8b5903 100644 --- a/TESTING.md +++ b/TESTING.md @@ -30,10 +30,10 @@ either mocked/patched in tests or are not unit-tested (see below). | Module | Status | Functions covered | Test file | |---|---|---|---| | `bolt/util.py` | Partial | `get_vcf_header_entry`, `get_vcf_header_line`, `get_qualified_vcf_annotation`, `add_vcf_header_entry`, `merge_tsv_files` | `tests/test_util.py` | -| `bolt/common/pcgr.py` | Partial | `get_ordering`, `get_impacts`, `determine_filter`, `get_variant_filter_data`, `split_vcf`, `run_somatic_chunk` (arg-mapping regression) | `tests/test_pcgr_hypermutated.py` | +| `bolt/common/pcgr.py` | Partial | `get_ordering`, `get_impacts`, `determine_filter`, `get_variant_filter_data`, `split_vcf`, `run_somatic_chunk` (arg-mapping regression) | `tests/test_pcgr.py` | | `bolt/common/pcgr.py` | Partial | `parse_genomic_change`, `get_impacts_higher`, `get_annotation_entry_tsv`, `compile_annotation_data`, `annotate_record` | `tests/test_pcgr_annotation.py` | | `bolt/workflows/smlv_somatic/filter.py` | Partial | `set_filter_data` | `tests/test_smlv_somatic_filter.py` | -| `bolt/workflows/smlv_somatic/report.py` | Partial | `select_pcgr_variants`, `count_variant_process`, `entry` overflow handling | `tests/test_pcgr_hypermutated.py` | +| `bolt/workflows/smlv_somatic/report.py` | Partial | `select_pcgr_variants`, `count_variant_process`, `entry` overflow handling | `tests/test_smlv_somatic_report.py` | "Partial" means the module has meaningful test coverage for its pure/testable logic, but not every function in the file is tested (see below for what is excluded and why). @@ -67,5 +67,5 @@ unit test suite: | `bolt/external/prioritize_sv.py` | Vendored third-party AstraZeneca SV annotation logic; no unit tests | Contributions raising coverage for these are welcome, but will require mocking -subprocess calls (as done in `tests/test_pcgr_hypermutated.py` for +subprocess calls (as done in `tests/test_smlv_somatic_report.py` for `select_pcgr_variants`) or dedicated fixture VCFs/TSVs under `tests/fixtures/`. diff --git a/tests/test_pcgr_annotation.py b/tests/test_pcgr_annotation.py index e62b97e..4014ece 100644 --- a/tests/test_pcgr_annotation.py +++ b/tests/test_pcgr_annotation.py @@ -2,7 +2,7 @@ Covers only binary-free / cyvcf2-in-memory logic. Functions requiring bcftools/PCGR/vcfanno subprocesses are intentionally NOT tested here. -Does NOT duplicate tests already in test_pcgr_hypermutated.py +Does NOT duplicate tests already in test_pcgr.py / test_smlv_somatic_report.py (get_ordering, get_impacts, determine_filter, get_variant_filter_data, split_vcf, run_somatic_chunk, count_variant_process, select_pcgr_variants). """ diff --git a/tests/test_smlv_somatic_report.py b/tests/test_smlv_somatic_report.py index 7953808..5c9f20e 100644 --- a/tests/test_smlv_somatic_report.py +++ b/tests/test_smlv_somatic_report.py @@ -14,7 +14,7 @@ import bolt.util as util import bolt.workflows.smlv_somatic.report as report_mod -from tests.helpers import HEADER, _csq, _write_vcf +from tests.helpers import _csq, _write_vcf class TestSelectPcgrVariants(unittest.TestCase): From 863b36e3b8cc21ea62c9235ebbee0e10e0112eba Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Mon, 13 Jul 2026 13:54:38 +1000 Subject: [PATCH 81/89] test(quick-j7e): add bcftools-guarded lossless regression test for merge_vcf_files Locks the sites-only invariant that keeps bcftools merge safe for the PCGR hypermutated chunk-merge path: TestMergeVcfFiles builds disjoint sites-only chunks, runs merge_vcf_files, and asserts no loss/dup, position-sorted, and indexed output. Guarded with skipUnless(shutil.which('bcftools')) so it runs in the conda CI env and skips cleanly elsewhere. Documents the rationale with a NOTE in merge_vcf_files and updates TESTING.md/CHANGELOG.md. --- CHANGELOG.md | 1 + TESTING.md | 11 +++-- bolt/util.py | 8 ++++ tests/test_util.py | 105 +++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 118 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e0c45a2..bf192c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## 0.3.2 +- Test: `TestMergeVcfFiles` — bcftools-guarded integration test proving `merge_vcf_files` (`bcftools merge -m all`) losslessly reassembles PCGR sites-only chunk VCFs (no loss/dup, position-sorted, indexed output). Locks the sites-only invariant that keeps `bcftools merge` safe for the hypermutated chunk-merge path; documented with a `NOTE` in `merge_vcf_files` - [32](https://github.com/umccr/bolt/pull/32) - Fix `PCGR_MUTATION_HOTSPOT=.` (dot placeholder) treated as truthy in retention check — was preventing tiered filtering from running for any sample with >450k PASS variants ([sash#52](https://github.com/umccr/sash/issues/52)) - [32](https://github.com/umccr/bolt/pull/32) - Graceful PCGR skip when `select_pcgr_variants` cannot cap variants to `MAX_SOMATIC_VARIANTS` — logs warning and continues without cancer report; non-PCGR outputs still publish ([sash#52](https://github.com/umccr/sash/issues/52)) - Fix: disable `--estimate_msi`/`--estimate_tmb` in chunked PCGR annotation runs — estimates on partial VCFs are not meaningful diff --git a/TESTING.md b/TESTING.md index e8b5903..5d4e5ae 100644 --- a/TESTING.md +++ b/TESTING.md @@ -20,16 +20,20 @@ Single test: python -m pytest tests/test_smlv_somatic_filter.py::TestSmlvSomaticFilter::test_min_af_filter -v ``` -All tests in `tests/` are pure Python + in-memory `cyvcf2` logic. They must run +Tests in `tests/` are pure Python + in-memory `cyvcf2` logic and must run without any bioinformatics binary installed (no `bcftools`, `pcgr`, `cpsr`, `vcfanno`, `snpEff`, `gpgr`, VEP). Functions that shell out to those tools are -either mocked/patched in tests or are not unit-tested (see below). +either mocked/patched in tests or are not unit-tested (see below). The one +exception is `TestMergeVcfFiles`, an integration test that exercises the real +`merge_vcf_files` → `bcftools merge` path; it is guarded with +`@unittest.skipUnless(shutil.which('bcftools'), ...)`, so it runs in the conda +CI env and skips cleanly (never fails) where `bcftools` is absent. ## Test Coverage | Module | Status | Functions covered | Test file | |---|---|---|---| -| `bolt/util.py` | Partial | `get_vcf_header_entry`, `get_vcf_header_line`, `get_qualified_vcf_annotation`, `add_vcf_header_entry`, `merge_tsv_files` | `tests/test_util.py` | +| `bolt/util.py` | Partial | `get_vcf_header_entry`, `get_vcf_header_line`, `get_qualified_vcf_annotation`, `add_vcf_header_entry`, `merge_tsv_files`, `merge_vcf_files` (bcftools-guarded lossless/sorted integration test) | `tests/test_util.py` | | `bolt/common/pcgr.py` | Partial | `get_ordering`, `get_impacts`, `determine_filter`, `get_variant_filter_data`, `split_vcf`, `run_somatic_chunk` (arg-mapping regression) | `tests/test_pcgr.py` | | `bolt/common/pcgr.py` | Partial | `parse_genomic_change`, `get_impacts_higher`, `get_annotation_entry_tsv`, `compile_annotation_data`, `annotate_record` | `tests/test_pcgr_annotation.py` | | `bolt/workflows/smlv_somatic/filter.py` | Partial | `set_filter_data` | `tests/test_smlv_somatic_filter.py` | @@ -47,7 +51,6 @@ unit test suite: | Function/module | Reason | |---|---| | `bolt/util.py: count_vcf_records` | Shells out to `bcftools view` | -| `bolt/util.py: merge_vcf_files` | Shells out to `bcftools merge`/`sort`/`index` | | `bolt/util.py: execute_command` | Spawns real subprocesses via `/bin/bash` | | `bolt/common/pcgr.py: prepare_vcf_somatic` / `prepare_vcf_germline` | Shells out to `bcftools index`/`bcftools view`/`bcftools annotate` | | `bolt/common/pcgr.py: run_somatic` / `run_somatic_chunk` (execution path) / `run_germline` | Invoke `pcgr`/`cpsr` CLI directly | diff --git a/bolt/util.py b/bolt/util.py index f837314..27ea6e1 100644 --- a/bolt/util.py +++ b/bolt/util.py @@ -138,6 +138,14 @@ def merge_tsv_files(tsv_files, merged_tsv_fp): def merge_vcf_files(vcf_files, merged_vcf_fp): + # NOTE(QC): reassembles PCGR hypermutated chunk outputs. `bcftools merge` is + # a multi-sample joiner and would fail with "Duplicate sample names" on + # same-named inputs that carry a genotype column — but this is safe here + # because PCGR inputs/outputs are sites-only (pcgr.prepare_vcf_somatic / + # get_minimal_header strip all FORMAT and sample columns), so merge just + # produces the union of disjoint chunks. Do not switch to a genotype-bearing + # VCF upstream without revisiting this. Locked by tests/test_util.py + # TestMergeVcfFiles (bcftools-guarded lossless/sorted regression test). merged_vcf_fp = pathlib.Path(merged_vcf_fp) merged_unsorted_vcf = merged_vcf_fp.parent / f'{merged_vcf_fp.name}.unsorted.vcf.gz' merged_vcf = merged_vcf_fp.parent / f'{merged_vcf_fp.name}.vcf.gz' diff --git a/tests/test_util.py b/tests/test_util.py index 446f2c4..7cac978 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1,10 +1,14 @@ -"""Unit tests for bolt/util.py — VCF header helpers and merge_tsv_files. +"""Unit tests for bolt/util.py — VCF header helpers and merge helpers. -Covers only binary-free logic. Functions requiring bcftools (count_vcf_records, -merge_vcf_files, execute_command) are intentionally NOT tested here. +Covers binary-free logic plus a bcftools-guarded integration test for +merge_vcf_files (see TestMergeVcfFiles). The remaining bcftools-dependent +functions (count_vcf_records, execute_command) are intentionally NOT tested +here. """ import gzip import pathlib +import shutil +import subprocess import tempfile import unittest @@ -180,5 +184,100 @@ def test_output_is_gzipped(self): self.assertEqual(magic, b'\x1f\x8b') +@unittest.skipUnless(shutil.which('bcftools'), 'bcftools not available') +class TestMergeVcfFiles(unittest.TestCase): + """Integration tests for util.merge_vcf_files(). + + merge_vcf_files reassembles PCGR hypermutated chunk outputs with + `bcftools merge -m all`. These chunks are sites-only VCFs (no FORMAT or + sample columns) because pcgr.prepare_vcf_somatic / get_minimal_header strip + them. bcftools merge only fails with "Duplicate sample names" when inputs + carry a same-named genotype column; on sites-only inputs it correctly + produces the union. These tests lock that invariant in: any regression that + reintroduces a sample column (which would break the merge) is caught here. + """ + + # Sites-only header (no FORMAT, no sample column) — mirrors get_minimal_header + _HEADER = ( + '##fileformat=VCFv4.2\n' + '##contig=\n' + '##contig=\n' + '##INFO=\n' + '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n' + ) + + def _write_chunk(self, path, records): + """Write a sites-only VCF, then bgzip + tabix-index it via bcftools. + + `records` is an iterable of (chrom, pos, ref, alt) tuples. Positions + within a chunk are sorted before writing so indexing succeeds; chunks + may be mutually out of order to exercise the cross-chunk sort. + """ + plain = pathlib.Path(f'{path}.plain.vcf') + with open(plain, 'w') as fh: + fh.write(self._HEADER) + for chrom, pos, ref, alt in sorted(records, key=lambda r: (r[0], r[1])): + fh.write(f'{chrom}\t{pos}\t.\t{ref}\t{alt}\t.\tPASS\tPCGR_TIER=1\n') + subprocess.run(['bcftools', 'view', '-Oz', '-o', str(path), str(plain)], check=True) + subprocess.run(['bcftools', 'index', '-t', str(path)], check=True) + return records + + def _read_keys(self, vcf_fp): + return [ + (record.CHROM, record.POS, record.REF, record.ALT[0]) + for record in cyvcf2.VCF(str(vcf_fp)) + ] + + def test_merge_is_lossless_and_sorted(self): + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + # Disjoint positions, and chunks deliberately out of order relative + # to each other so the merge must interleave/sort across chunks. + chunk_a = self._write_chunk( + tmp_path / 'chunk_a.vcf.gz', + [('chr1', 300, 'G', 'A'), ('chr1', 100, 'A', 'T'), ('chr2', 50, 'C', 'G')], + ) + chunk_b = self._write_chunk( + tmp_path / 'chunk_b.vcf.gz', + [('chr1', 200, 'C', 'G'), ('chr1', 400, 'T', 'C')], + ) + + merged_vcf = util.merge_vcf_files( + [tmp_path / 'chunk_a.vcf.gz', tmp_path / 'chunk_b.vcf.gz'], + tmp_path / 'merged.pass', + ) + + merged_keys = self._read_keys(merged_vcf) + expected_keys = list(chunk_a) + list(chunk_b) + + # No loss, no duplication: exact multiset match + self.assertEqual(len(merged_keys), len(expected_keys)) + self.assertCountEqual(merged_keys, expected_keys) + # Position-sorted output (cross-chunk interleave) + self.assertEqual( + merged_keys, + sorted(merged_keys, key=lambda k: (k[0], k[1])), + ) + + def test_merge_output_is_bgzipped_and_indexed(self): + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + self._write_chunk(tmp_path / 'chunk_a.vcf.gz', [('chr1', 100, 'A', 'T')]) + self._write_chunk(tmp_path / 'chunk_b.vcf.gz', [('chr1', 200, 'C', 'G')]) + + merged_vcf = util.merge_vcf_files( + [tmp_path / 'chunk_a.vcf.gz', tmp_path / 'chunk_b.vcf.gz'], + tmp_path / 'merged.pass', + ) + + # BGZF/gzip magic bytes + with open(merged_vcf, 'rb') as fh: + self.assertEqual(fh.read(2), b'\x1f\x8b') + # merge_vcf_files tabix-indexes its output + self.assertTrue(pathlib.Path(f'{merged_vcf}.tbi').exists()) + # Intermediate unsorted file is cleaned up + self.assertFalse((tmp_path / 'merged.pass.unsorted.vcf.gz').exists()) + + if __name__ == '__main__': unittest.main() From 7cd15afb61f840e33b56a83dd395fca7d5f650c5 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Mon, 13 Jul 2026 14:54:13 +1000 Subject: [PATCH 82/89] fix(quick-p3k): handle duplicate PCGR VCF keys in get_annotations_vcf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports the warn-and-skip fix from oa-only-v2 (fddc7a7), which was never merged into release/0.3.0. PCGR can emit duplicate variant entries in its output VCF when a variant maps to multiple transcripts (sash sample L2600141 hit this). The assert crashed annotate; now duplicates are logged and the first entry is kept, consistent with the TSV-side dedup. Covered by TestGetAnnotationsVcf in tests/test_pcgr_annotation.py — verified to fail with AssertionError against the pre-fix code. --- bolt/common/pcgr.py | 8 +++++- tests/test_pcgr_annotation.py | 52 +++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/bolt/common/pcgr.py b/bolt/common/pcgr.py index 019be83..9338b3f 100644 --- a/bolt/common/pcgr.py +++ b/bolt/common/pcgr.py @@ -434,7 +434,13 @@ def get_annotations_vcf(vcf_fp, info_field_map): assert len(record.ALT) == 1 [alt] = record.ALT key = (f'chr{record.CHROM}', record.POS, record.REF, alt) - assert key not in data_vcf + + # NOTE(QC): PCGR can emit duplicate variant entries in its output VCF when a + # variant maps to multiple transcripts (sash sample L2600141). Keep the first + # entry and warn rather than asserting, consistent with the TSV-side dedup. + if key in data_vcf: + logger.warning(f'Duplicate PCGR VCF key {key}: keeping first entry') + continue data_vcf[key] = dict() for info_dst, info_src in info_field_map.items(): diff --git a/tests/test_pcgr_annotation.py b/tests/test_pcgr_annotation.py index 4014ece..adece51 100644 --- a/tests/test_pcgr_annotation.py +++ b/tests/test_pcgr_annotation.py @@ -262,5 +262,57 @@ def test_key_uses_chrom_pos_ref_alt(self): ) +class TestGetAnnotationsVcf(unittest.TestCase): + """Unit tests for pcgr.get_annotations_vcf() duplicate-key handling. + + PCGR strips the 'chr' prefix from its own output VCF, so fixtures here use + bare contig names ('1', not 'chr1') to match real PCGR output. + """ + + PCGR_VCF_HEADER = ( + '##fileformat=VCFv4.2\n' + '##FILTER=\n' + f'##INFO=\n' + '##contig=\n' + '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n' + ) + + def _write_pcgr_vcf(self, path, rows): + with open(path, 'w') as fh: + fh.write(self.PCGR_VCF_HEADER) + for chrom, pos, ref, alt, info in rows: + fh.write(f'{chrom}\t{pos}\t.\t{ref}\t{alt}\t.\tPASS\t{info}\n') + + def test_duplicate_vcf_key_keeps_first_no_crash(self): + """A duplicate variant in PCGR's VCF output must be skipped, not raise AssertionError.""" + info_field_map = {constants.VcfInfo.PCGR_CSQ: 'PCGR_CSQ'} + with tempfile.TemporaryDirectory() as tmp: + vcf_fp = pathlib.Path(tmp) / 'pcgr.vcf' + self._write_pcgr_vcf(vcf_fp, [ + ('1', 100, 'A', 'T', 'PCGR_CSQ=first'), + ('1', 100, 'A', 'T', 'PCGR_CSQ=second'), # duplicate transcript mapping + ]) + result = pcgr.get_annotations_vcf(vcf_fp, info_field_map) + + key = ('chr1', 100, 'A', 'T') + self.assertEqual(len(result), 1) + self.assertEqual(result[key][constants.VcfInfo.PCGR_CSQ], 'first') + + def test_non_duplicate_vcf_keys_all_present(self): + """Distinct variants are all retained unaffected by dedup handling.""" + info_field_map = {constants.VcfInfo.PCGR_CSQ: 'PCGR_CSQ'} + with tempfile.TemporaryDirectory() as tmp: + vcf_fp = pathlib.Path(tmp) / 'pcgr.vcf' + self._write_pcgr_vcf(vcf_fp, [ + ('1', 100, 'A', 'T', 'PCGR_CSQ=v1'), + ('1', 200, 'C', 'G', 'PCGR_CSQ=v2'), + ]) + result = pcgr.get_annotations_vcf(vcf_fp, info_field_map) + + self.assertEqual(len(result), 2) + self.assertEqual(result[('chr1', 100, 'A', 'T')][constants.VcfInfo.PCGR_CSQ], 'v1') + self.assertEqual(result[('chr1', 200, 'C', 'G')][constants.VcfInfo.PCGR_CSQ], 'v2') + + if __name__ == '__main__': unittest.main() From 24b1206cc13671ba08bb14a724865aa5a446fc8f Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Mon, 13 Jul 2026 14:58:12 +1000 Subject: [PATCH 83/89] docs: link get_annotations_vcf dup-key NOTE to bolt#34 instead of sample ID Sample identifiers shouldn't live in source comments. fddc7a7 (the fix this NOTE documents) landed as part of umccr/bolt#34. --- bolt/common/pcgr.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bolt/common/pcgr.py b/bolt/common/pcgr.py index 9338b3f..970665e 100644 --- a/bolt/common/pcgr.py +++ b/bolt/common/pcgr.py @@ -436,8 +436,9 @@ def get_annotations_vcf(vcf_fp, info_field_map): key = (f'chr{record.CHROM}', record.POS, record.REF, alt) # NOTE(QC): PCGR can emit duplicate variant entries in its output VCF when a - # variant maps to multiple transcripts (sash sample L2600141). Keep the first - # entry and warn rather than asserting, consistent with the TSV-side dedup. + # variant maps to multiple transcripts. Keep the first entry and warn rather + # than asserting, consistent with the TSV-side dedup. + # https://github.com/umccr/bolt/pull/34 if key in data_vcf: logger.warning(f'Duplicate PCGR VCF key {key}: keeping first entry') continue From 39dc084ed68711f42e3818c9f4501d1320bbecf1 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Mon, 13 Jul 2026 15:00:34 +1000 Subject: [PATCH 84/89] fix(quick-r8m): handle duplicate PCGR/CPSR TSV keys instead of asserting Same bug class as quick-p3k's get_annotations_vcf fix, on the sibling TSV-parsing paths. Both were still unfixed on release/0.3.0: - collect_pcgr_annotation_data (somatic): the tier-aware dedup fix for this existed on another branch but never landed here. Extracted tier normalisation into _normalise_tier(), added _TIER_ORDER, and on a duplicate key keep whichever entry has the more actionable (lower) tier, warning either way. - collect_cpsr_annotation_data (germline): no fix existed anywhere. CPSR TSV rows carry no actionability tier to break ties on, so this uses warn-and-keep-first, matching get_annotations_vcf's approach. Covered by TestCollectPcgrAnnotationData and TestCollectCpsrAnnotationData in tests/test_pcgr_annotation.py. Regression-proofed: 3 of the 5 new tests fail with AssertionError against the pre-fix code (the 2 that don't hit the assert path correctly pass either way). --- bolt/common/pcgr.py | 55 ++++++++++++----- tests/test_pcgr_annotation.py | 110 ++++++++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+), 16 deletions(-) diff --git a/bolt/common/pcgr.py b/bolt/common/pcgr.py index 970665e..b8989cf 100644 --- a/bolt/common/pcgr.py +++ b/bolt/common/pcgr.py @@ -351,6 +351,25 @@ def transfer_annotations_germline(input_fp, normal_name, cpsr_dir, output_dir): output_fh.close() +# Lower value = more clinically actionable. Used to resolve duplicate PCGR TSV +# rows for the same variant (PCGR can emit one row per transcript mapping). +_TIER_ORDER = {'1': 0, '2': 1, '3': 2, '4': 3, 'N': 4} + + +def _normalise_tier(raw_tier): + # Normalize PCGR actionability tier to simple values: '1','2','3','4','N' + tier_norm = (raw_tier or '').strip().replace('_', ' ').upper() + if tier_norm in ('TIER 1', 'TIER1', '1'): + return '1' + elif tier_norm in ('TIER 2', 'TIER2', '2'): + return '2' + elif tier_norm in ('TIER 3', 'TIER3', '3'): + return '3' + elif tier_norm in ('TIER 4', 'TIER4', '4'): + return '4' + return 'N' + + def collect_pcgr_annotation_data(tsv_fp, vcf_fp, info_field_map): # Gather all annotations from TSV data_tsv = dict() @@ -359,23 +378,19 @@ def collect_pcgr_annotation_data(tsv_fp, vcf_fp, info_field_map): with open_fn(tsv_fp, 'rt') as tsv_fh: for record in csv.DictReader(tsv_fh, delimiter='\t'): key, record_ann = get_annotation_entry_tsv(record, info_field_map) - assert key not in data_tsv - - # Normalize PCGR actionability tier to simple values: '1','2','3','4','N' - raw_tier = (record.get('ACTIONABILITY_TIER') or '').strip() - tier_norm = raw_tier.replace('_', ' ').upper() - if tier_norm in ('TIER 1','TIER1','1'): - tier_val = '1' - elif tier_norm in ('TIER 2','TIER2','2'): - tier_val = '2' - elif tier_norm in ('TIER 3','TIER3','3'): - tier_val = '3' - elif tier_norm in ('TIER 4','TIER4','4'): - tier_val = '4' - else: - tier_val = 'N' + tier_val = _normalise_tier(record.get('ACTIONABILITY_TIER')) record_ann[constants.VcfInfo.PCGR_ACTIONABILITY_TIER] = tier_val + # NOTE(QC): PCGR can emit multiple TSV rows for the same variant when it + # maps to multiple transcripts. Keep the most actionable entry (lowest + # tier number) rather than asserting. See https://github.com/umccr/bolt/pull/34 + if key in data_tsv: + existing_tier = data_tsv[key][constants.VcfInfo.PCGR_ACTIONABILITY_TIER] + if _TIER_ORDER[tier_val] >= _TIER_ORDER[existing_tier]: + logger.warning(f'Duplicate PCGR TSV key {key}: keeping tier {existing_tier}, skipping tier {tier_val}') + continue + logger.warning(f'Duplicate PCGR TSV key {key}: replacing tier {existing_tier} with more actionable tier {tier_val}') + # Store annotation data data_tsv[key] = record_ann @@ -403,7 +418,15 @@ def collect_cpsr_annotation_data(tsv_fp, vcf_fp, info_field_map): record['ALT'] = re_result.group('alt') key, record_ann = get_annotation_entry_tsv(record, info_field_map) - assert key not in data_tsv + + # NOTE(QC): CPSR can emit multiple TSV rows for the same variant when it + # maps to multiple transcripts (same class of issue as the PCGR somatic + # TSV path above). CPSR rows carry no actionability tier to break ties + # on, so keep the first entry and warn. See https://github.com/umccr/bolt/pull/34 + if key in data_tsv: + logger.warning(f'Duplicate CPSR TSV key {key}: keeping first entry') + continue + data_tsv[key] = record_ann # Gather annotations from VCF diff --git a/tests/test_pcgr_annotation.py b/tests/test_pcgr_annotation.py index adece51..17b2ec8 100644 --- a/tests/test_pcgr_annotation.py +++ b/tests/test_pcgr_annotation.py @@ -314,5 +314,115 @@ def test_non_duplicate_vcf_keys_all_present(self): self.assertEqual(result[('chr1', 200, 'C', 'G')][constants.VcfInfo.PCGR_CSQ], 'v2') +def _write_empty_pcgr_vcf(path): + """Header-only sites-only VCF, so get_annotations_vcf() returns {}.""" + with open(path, 'w') as fh: + fh.write( + '##fileformat=VCFv4.2\n' + '##FILTER=\n' + f'##INFO=\n' + '##contig=\n' + '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n' + ) + + +def _write_tsv(path, header, rows): + import csv as _csv + with open(path, 'w', newline='') as fh: + writer = _csv.DictWriter(fh, fieldnames=header, delimiter='\t') + writer.writeheader() + for row in rows: + writer.writerow(row) + + +class TestCollectPcgrAnnotationData(unittest.TestCase): + """Unit tests for pcgr.collect_pcgr_annotation_data() duplicate-key tier resolution.""" + + HEADER = ['GENOMIC_CHANGE', 'CSQ', 'ACTIONABILITY_TIER'] + + def _collect(self, tmp, rows): + tsv_fp = pathlib.Path(tmp) / 'pcgr.tsv' + vcf_fp = pathlib.Path(tmp) / 'pcgr.vcf' + _write_tsv(tsv_fp, self.HEADER, rows) + _write_empty_pcgr_vcf(vcf_fp) + info_field_map = {constants.VcfInfo.PCGR_CSQ: 'CSQ'} + return pcgr.collect_pcgr_annotation_data(tsv_fp, vcf_fp, info_field_map) + + def test_duplicate_key_keeps_more_actionable_tier_no_crash(self): + """A second, more-actionable-tier row for the same variant replaces the first.""" + with tempfile.TemporaryDirectory() as tmp: + result = self._collect(tmp, [ + {'GENOMIC_CHANGE': '1:g.100A>T', 'CSQ': 'transcript_a', 'ACTIONABILITY_TIER': 'TIER 3'}, + {'GENOMIC_CHANGE': '1:g.100A>T', 'CSQ': 'transcript_b', 'ACTIONABILITY_TIER': 'TIER 1'}, + ]) + key = ('chr1', 100, 'A', 'T') + self.assertEqual(len(result), 1) + self.assertEqual(result[key][constants.VcfInfo.PCGR_CSQ], 'transcript_b') + self.assertEqual(result[key][constants.VcfInfo.PCGR_ACTIONABILITY_TIER], '1') + + def test_duplicate_key_skips_less_actionable_tier(self): + """A second, less-actionable-tier row for the same variant is dropped.""" + with tempfile.TemporaryDirectory() as tmp: + result = self._collect(tmp, [ + {'GENOMIC_CHANGE': '1:g.100A>T', 'CSQ': 'transcript_a', 'ACTIONABILITY_TIER': 'TIER 1'}, + {'GENOMIC_CHANGE': '1:g.100A>T', 'CSQ': 'transcript_b', 'ACTIONABILITY_TIER': 'TIER 3'}, + ]) + key = ('chr1', 100, 'A', 'T') + self.assertEqual(len(result), 1) + self.assertEqual(result[key][constants.VcfInfo.PCGR_CSQ], 'transcript_a') + self.assertEqual(result[key][constants.VcfInfo.PCGR_ACTIONABILITY_TIER], '1') + + def test_non_duplicate_keys_all_present(self): + with tempfile.TemporaryDirectory() as tmp: + result = self._collect(tmp, [ + {'GENOMIC_CHANGE': '1:g.100A>T', 'CSQ': 'v1', 'ACTIONABILITY_TIER': 'TIER 1'}, + {'GENOMIC_CHANGE': '1:g.200C>G', 'CSQ': 'v2', 'ACTIONABILITY_TIER': 'TIER 2'}, + ]) + self.assertEqual(len(result), 2) + self.assertIn(('chr1', 100, 'A', 'T'), result) + self.assertIn(('chr1', 200, 'C', 'G'), result) + + +class TestCollectCpsrAnnotationData(unittest.TestCase): + """Unit tests for pcgr.collect_cpsr_annotation_data() duplicate-key handling.""" + + HEADER = ['GENOMIC_CHANGE', 'CSQ'] + + def _collect(self, tmp, rows): + tsv_fp = pathlib.Path(tmp) / 'cpsr.tsv.gz' + vcf_fp = pathlib.Path(tmp) / 'cpsr.vcf' + import gzip as _gzip + import csv as _csv + with _gzip.open(tsv_fp, 'wt', newline='') as fh: + writer = _csv.DictWriter(fh, fieldnames=self.HEADER, delimiter='\t') + writer.writeheader() + for row in rows: + writer.writerow(row) + _write_empty_pcgr_vcf(vcf_fp) + info_field_map = {constants.VcfInfo.PCGR_CSQ: 'CSQ'} + return pcgr.collect_cpsr_annotation_data(tsv_fp, vcf_fp, info_field_map) + + def test_duplicate_key_keeps_first_no_crash(self): + """A duplicate CPSR TSV row for the same variant must be skipped, not raise.""" + with tempfile.TemporaryDirectory() as tmp: + result = self._collect(tmp, [ + {'GENOMIC_CHANGE': '1:g.100A>T', 'CSQ': 'first'}, + {'GENOMIC_CHANGE': '1:g.100A>T', 'CSQ': 'second'}, + ]) + key = ('chr1', 100, 'A', 'T') + self.assertEqual(len(result), 1) + self.assertEqual(result[key][constants.VcfInfo.PCGR_CSQ], 'first') + + def test_non_duplicate_keys_all_present(self): + with tempfile.TemporaryDirectory() as tmp: + result = self._collect(tmp, [ + {'GENOMIC_CHANGE': '1:g.100A>T', 'CSQ': 'v1'}, + {'GENOMIC_CHANGE': '1:g.200C>G', 'CSQ': 'v2'}, + ]) + self.assertEqual(len(result), 2) + self.assertIn(('chr1', 100, 'A', 'T'), result) + self.assertIn(('chr1', 200, 'C', 'G'), result) + + if __name__ == '__main__': unittest.main() From 672763ada517cc201bf3641a17f296b68b3c309d Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Mon, 13 Jul 2026 15:01:14 +1000 Subject: [PATCH 85/89] docs: add CHANGELOG entries for the duplicate-key assert fixes --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf192c7..93b6660 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## 0.3.2 +- Fix: `get_annotations_vcf` crashed with `AssertionError` on a duplicate PCGR VCF key (variant mapping to multiple transcripts) — now logs a warning and keeps the first entry ([34](https://github.com/umccr/bolt/pull/34)) +- Fix: `collect_pcgr_annotation_data` / `collect_cpsr_annotation_data` had the same duplicate-key `AssertionError` on their TSV paths — somatic path now keeps the most actionable tier on a duplicate, germline/CPSR path keeps the first entry; both warn instead of crashing +- Test: `TestGetAnnotationsVcf`, `TestCollectPcgrAnnotationData`, `TestCollectCpsrAnnotationData` — duplicate-key regression coverage for all three annotation-collection paths - Test: `TestMergeVcfFiles` — bcftools-guarded integration test proving `merge_vcf_files` (`bcftools merge -m all`) losslessly reassembles PCGR sites-only chunk VCFs (no loss/dup, position-sorted, indexed output). Locks the sites-only invariant that keeps `bcftools merge` safe for the hypermutated chunk-merge path; documented with a `NOTE` in `merge_vcf_files` - [32](https://github.com/umccr/bolt/pull/32) - Fix `PCGR_MUTATION_HOTSPOT=.` (dot placeholder) treated as truthy in retention check — was preventing tiered filtering from running for any sample with >450k PASS variants ([sash#52](https://github.com/umccr/sash/issues/52)) - [32](https://github.com/umccr/bolt/pull/32) - Graceful PCGR skip when `select_pcgr_variants` cannot cap variants to `MAX_SOMATIC_VARIANTS` — logs warning and continues without cancer report; non-PCGR outputs still publish ([sash#52](https://github.com/umccr/sash/issues/52)) From 26ea34f1909a5911c66595e4d39911477178f0ef Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Mon, 13 Jul 2026 15:05:54 +1000 Subject: [PATCH 86/89] fix(quick-v4t): disable PCGR estimates for hypermutated tiered-selection path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit report.py's entry() calls pcgr.run_somatic() for hypermutated samples via select_pcgr_variants (tiered selection), but never passed disable_estimates=True — unlike run_somatic_chunk, which always disables estimates for its equally-partial chunked inputs. TMB/MSI computed on a tiered-selection subset are not meaningful and the report gave no indication they were derived from a trimmed variant set. Tracks pcgr_input_trimmed (the same condition already used to decide whether select_pcgr_variants runs at all) and passes it through as disable_estimates. Covered by two assertions in TestEntrySkipsPcgrOnOverflow: the trimmed path gets disable_estimates=True (new test), the untrimmed path keeps disable_estimates=False (extended existing test). Both fail with KeyError against the pre-fix code (disable_estimates kwarg absent). --- bolt/workflows/smlv_somatic/report.py | 7 ++++++- tests/test_smlv_somatic_report.py | 30 +++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/bolt/workflows/smlv_somatic/report.py b/bolt/workflows/smlv_somatic/report.py index 2501b54..e026cdc 100644 --- a/bolt/workflows/smlv_somatic/report.py +++ b/bolt/workflows/smlv_somatic/report.py @@ -116,7 +116,8 @@ def entry(ctx, **kwargs): purple_data = parse_purple_purity_file(kwargs['purple_purity_fp']) pcgr_skipped = False - if variant_counts_process['filter_pass'] <= constants.MAX_SOMATIC_VARIANTS: + pcgr_input_trimmed = variant_counts_process['filter_pass'] > constants.MAX_SOMATIC_VARIANTS + if not pcgr_input_trimmed: pcgr_input_vcf_fp = kwargs['vcf_fp'] else: try: @@ -153,6 +154,10 @@ def entry(ctx, **kwargs): purity=purple_data['purity'], ploidy=purple_data['ploidy'], sample_id=kwargs['tumor_name'], + # NOTE(QC): pcgr_input_vcf_fp is a tiered-selection subset when + # pcgr_input_trimmed, so TMB/MSI estimates on it are not meaningful — + # same reasoning as run_somatic_chunk's chunked inputs. + disable_estimates=pcgr_input_trimmed, ) diff --git a/tests/test_smlv_somatic_report.py b/tests/test_smlv_somatic_report.py index 5c9f20e..78dd9c2 100644 --- a/tests/test_smlv_somatic_report.py +++ b/tests/test_smlv_somatic_report.py @@ -381,6 +381,36 @@ def test_run_somatic_called_when_within_limit(self): self.assertEqual(result.exit_code, 0, result.output) mock_run.assert_called_once() + self.assertIs(mock_run.call_args.kwargs['disable_estimates'], False) + + def test_run_somatic_called_with_disable_estimates_when_trimmed(self): + """When select_pcgr_variants trims a hypermutated sample, estimates must be disabled.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + dummy = tmp_path / 'dummy.vcf.gz' + dummy.touch() + fake_selected_output = tmp_path / 'selected.vcf.gz' + fake_selected_output.touch() + fake_prep_output = tmp_path / 'prep.vcf.gz' + fake_prep_output.touch() + + with patch.object(report_mod, 'bcftools_stats_prepare', return_value=dummy), \ + patch.object(report_mod, 'run_bcftools_stats'), \ + patch.object(report_mod, 'allele_frequencies'), \ + patch.object(report_mod, 'count_variant_types', return_value=_PASS_COUNTS), \ + patch.object(report_mod, 'count_variant_process', + return_value={'filter_pass': constants.MAX_SOMATIC_VARIANTS + 1}), \ + patch.object(report_mod, 'parse_purple_purity_file', + return_value={'purity': 0.8, 'ploidy': 2.0}), \ + patch.object(report_mod, 'select_pcgr_variants', + return_value=fake_selected_output), \ + patch.object(pcgr, 'prepare_vcf_somatic', return_value=fake_prep_output), \ + patch.object(pcgr, 'run_somatic') as mock_run: + result = CliRunner().invoke(report_mod.entry, _cli_args(dummy, tmp_path / 'out')) + + self.assertEqual(result.exit_code, 0, result.output) + mock_run.assert_called_once() + self.assertIs(mock_run.call_args.kwargs['disable_estimates'], True) if __name__ == '__main__': From b0e8f8ef2599411dcf7aaf284a347c9214fe06c9 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Mon, 13 Jul 2026 15:06:39 +1000 Subject: [PATCH 87/89] docs: add CHANGELOG entry for hypermutated disable_estimates fix --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93b6660..9bc2ef0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## 0.3.2 +- Fix: hypermutated `report` samples (tiered-selection path) never disabled `--estimate_msi`/`--estimate_tmb` — TMB/MSI were computed on a trimmed variant subset with no indication in the report. Now passes `disable_estimates=True` when `select_pcgr_variants` trims input, matching `run_somatic_chunk`'s existing chunked-input behavior - Fix: `get_annotations_vcf` crashed with `AssertionError` on a duplicate PCGR VCF key (variant mapping to multiple transcripts) — now logs a warning and keeps the first entry ([34](https://github.com/umccr/bolt/pull/34)) - Fix: `collect_pcgr_annotation_data` / `collect_cpsr_annotation_data` had the same duplicate-key `AssertionError` on their TSV paths — somatic path now keeps the most actionable tier on a duplicate, germline/CPSR path keeps the first entry; both warn instead of crashing - Test: `TestGetAnnotationsVcf`, `TestCollectPcgrAnnotationData`, `TestCollectCpsrAnnotationData` — duplicate-key regression coverage for all three annotation-collection paths From 1afb683ca22c1e434a73aef757e22ef85f78829f Mon Sep 17 00:00:00 2001 From: Quentin Clayssen Date: Mon, 13 Jul 2026 15:15:49 +1000 Subject: [PATCH 88/89] fix(quick-w2n): check SAGE VCF headers against sage_vcf_fp, not input_fp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rescue.py's annotate_existing_sage_calls called util.check_annotation_headers(info_field_map_sage, input_fp) — but input_fp is the DRAGEN VCF being annotated, not the SAGE VCF the check is meant to validate. Introduced in PR #24 (commit 6a93f6b, 2025-11-24), implementing a TODO that explicitly asked to check headers against the SAGE fp. Because check_annotation_headers silently skips any field absent from its target VCF, and the DRAGEN VCF has none of the SAGE_* fields, the check ran but validated nothing. Not on main. Also adds first-ever test coverage for rescue.py and for util.check_annotation_headers, which had none: - tests/test_util.py: TestCheckAnnotationHeaders — matching description passes, mismatched description exits, field absent from target VCF is skipped. - tests/test_smlv_somatic_rescue.py (new): regression test proving the header check now reads sage_vcf_fp. Regression-proofed: against the pre-fix code the test fails (not with a clean assertion failure, but a downstream KeyError from FORMAT/AD access — proving the check silently passed and execution continued straight past it). Updates TESTING.md (rescue.py, get_annotations_vcf, collect_pcgr_annotation_data, collect_cpsr_annotation_data coverage rows were stale from earlier quick tasks) and CHANGELOG.md. --- CHANGELOG.md | 1 + TESTING.md | 10 ++-- bolt/workflows/smlv_somatic/rescue.py | 2 +- tests/test_smlv_somatic_rescue.py | 75 +++++++++++++++++++++++++++ tests/test_util.py | 54 +++++++++++++++++++ 5 files changed, 136 insertions(+), 6 deletions(-) create mode 100644 tests/test_smlv_somatic_rescue.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bc2ef0..0b43c32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## 0.3.2 +- Fix: `rescue.py`'s SAGE VCF header-consistency check compared the DRAGEN input VCF's headers against the expected SAGE header descriptions instead of the actual SAGE VCF's — the check ran but validated nothing (introduced [24](https://github.com/umccr/bolt/pull/24)). Now checks `sage_vcf_fp`. First-ever test coverage added for `rescue.py` and for `util.check_annotation_headers` - Fix: hypermutated `report` samples (tiered-selection path) never disabled `--estimate_msi`/`--estimate_tmb` — TMB/MSI were computed on a trimmed variant subset with no indication in the report. Now passes `disable_estimates=True` when `select_pcgr_variants` trims input, matching `run_somatic_chunk`'s existing chunked-input behavior - Fix: `get_annotations_vcf` crashed with `AssertionError` on a duplicate PCGR VCF key (variant mapping to multiple transcripts) — now logs a warning and keeps the first entry ([34](https://github.com/umccr/bolt/pull/34)) - Fix: `collect_pcgr_annotation_data` / `collect_cpsr_annotation_data` had the same duplicate-key `AssertionError` on their TSV paths — somatic path now keeps the most actionable tier on a duplicate, germline/CPSR path keeps the first entry; both warn instead of crashing diff --git a/TESTING.md b/TESTING.md index 5d4e5ae..1676ea2 100644 --- a/TESTING.md +++ b/TESTING.md @@ -33,11 +33,12 @@ CI env and skips cleanly (never fails) where `bcftools` is absent. | Module | Status | Functions covered | Test file | |---|---|---|---| -| `bolt/util.py` | Partial | `get_vcf_header_entry`, `get_vcf_header_line`, `get_qualified_vcf_annotation`, `add_vcf_header_entry`, `merge_tsv_files`, `merge_vcf_files` (bcftools-guarded lossless/sorted integration test) | `tests/test_util.py` | +| `bolt/util.py` | Partial | `get_vcf_header_entry`, `get_vcf_header_line`, `get_qualified_vcf_annotation`, `add_vcf_header_entry`, `merge_tsv_files`, `merge_vcf_files` (bcftools-guarded lossless/sorted integration test), `check_annotation_headers` | `tests/test_util.py` | | `bolt/common/pcgr.py` | Partial | `get_ordering`, `get_impacts`, `determine_filter`, `get_variant_filter_data`, `split_vcf`, `run_somatic_chunk` (arg-mapping regression) | `tests/test_pcgr.py` | -| `bolt/common/pcgr.py` | Partial | `parse_genomic_change`, `get_impacts_higher`, `get_annotation_entry_tsv`, `compile_annotation_data`, `annotate_record` | `tests/test_pcgr_annotation.py` | +| `bolt/common/pcgr.py` | Partial | `parse_genomic_change`, `get_impacts_higher`, `get_annotation_entry_tsv`, `compile_annotation_data`, `annotate_record`, `get_annotations_vcf` (duplicate-key regression), `collect_pcgr_annotation_data` (duplicate-key tier resolution), `collect_cpsr_annotation_data` (duplicate-key regression) | `tests/test_pcgr_annotation.py` | | `bolt/workflows/smlv_somatic/filter.py` | Partial | `set_filter_data` | `tests/test_smlv_somatic_filter.py` | -| `bolt/workflows/smlv_somatic/report.py` | Partial | `select_pcgr_variants`, `count_variant_process`, `entry` overflow handling | `tests/test_smlv_somatic_report.py` | +| `bolt/workflows/smlv_somatic/report.py` | Partial | `select_pcgr_variants`, `count_variant_process`, `entry` overflow handling, `entry` `disable_estimates` branching | `tests/test_smlv_somatic_report.py` | +| `bolt/workflows/smlv_somatic/rescue.py` | Partial | `annotate_existing_sage_calls` (SAGE VCF header-consistency check only) | `tests/test_smlv_somatic_rescue.py` | "Partial" means the module has meaningful test coverage for its pure/testable logic, but not every function in the file is tested (see below for what is excluded and why). @@ -55,10 +56,9 @@ unit test suite: | `bolt/common/pcgr.py: prepare_vcf_somatic` / `prepare_vcf_germline` | Shells out to `bcftools index`/`bcftools view`/`bcftools annotate` | | `bolt/common/pcgr.py: run_somatic` / `run_somatic_chunk` (execution path) / `run_germline` | Invoke `pcgr`/`cpsr` CLI directly | | `bolt/common/pcgr.py: transfer_annotations_somatic` / `transfer_annotations_germline` | Depend on real PCGR/CPSR TSV+VCF output files | -| `bolt/common/pcgr.py: collect_pcgr_annotation_data` / `collect_cpsr_annotation_data` / `get_annotations_vcf` | Read real PCGR/CPSR TSV/VCF fixture files end-to-end (only their pure sub-helpers are unit tested — see `get_annotation_entry_tsv`, `compile_annotation_data`) | | `bolt/common/pcgr.py: merging_pcgr_files` | Wraps `merge_vcf_files`/`merge_tsv_files` (bcftools-dependent) | | `bolt/workflows/smlv_somatic/annotate.py` | Orchestrates vcfanno + PON + PCGR subprocess pipeline; no unit tests | -| `bolt/workflows/smlv_somatic/rescue.py` | SAGE hotspot recall logic; no unit tests | +| `bolt/workflows/smlv_somatic/rescue.py` (all functions except the header check) | SAGE hotspot recall orchestrates `bcftools isec`/`concat`/`annotate` subprocesses end-to-end; no unit tests | | `bolt/workflows/smlv_somatic/prepare.py` | bcftools-based VCF prep; no unit tests | | `bolt/workflows/smlv_germline/prepare.py` | bcftools-based panel region selection; no unit tests | | `bolt/workflows/smlv_germline/report.py` | bcftools stats + CPSR report generation; no unit tests | diff --git a/bolt/workflows/smlv_somatic/rescue.py b/bolt/workflows/smlv_somatic/rescue.py index ff69e6e..e544f72 100644 --- a/bolt/workflows/smlv_somatic/rescue.py +++ b/bolt/workflows/smlv_somatic/rescue.py @@ -125,7 +125,7 @@ def annotate_existing_sage_calls(input_fp, tumor_name, sage_vcf_fp, output_dir): constants.VcfFilter.SAGE_LOWCONF: 'SAGE_LOWCONF', } - util.check_annotation_headers(info_field_map_sage, input_fp) + util.check_annotation_headers(info_field_map_sage, sage_vcf_fp) # Add header entries so that they are included in the output file via templating done below util.add_vcf_header_entry(input_fh, constants.VcfFilter.SAGE_LOWCONF) diff --git a/tests/test_smlv_somatic_rescue.py b/tests/test_smlv_somatic_rescue.py new file mode 100644 index 0000000..cd66902 --- /dev/null +++ b/tests/test_smlv_somatic_rescue.py @@ -0,0 +1,75 @@ +"""Unit tests for bolt/workflows/smlv_somatic/rescue.py. + +Covers only the SAGE VCF header-consistency check in +annotate_existing_sage_calls(), which is reachable without a bcftools binary +(the check runs before any subprocess call — a SystemExit from a mismatch +short-circuits execution before bcftools is ever invoked). The rest of +rescue.py orchestrates bcftools subprocesses end-to-end and is not +unit-tested here. +""" +import pathlib +import tempfile +import unittest + +import bolt.common.constants as constants +import bolt.workflows.smlv_somatic.rescue as rescue + + +def _write_input_vcf_no_sage_fields(path): + """A plain DRAGEN-style VCF with no SAGE_* header lines at all.""" + with open(path, 'w') as fh: + fh.write( + '##fileformat=VCFv4.2\n' + '##FILTER=\n' + '##FORMAT=\n' + '##contig=\n' + '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tTUMOR\n' + 'chr1\t100\t.\tA\tT\t.\tPASS\t.\tAD\t10,5\n' + ) + + +def _write_sage_vcf_mismatched_hotspot_description(path): + """A SAGE-style VCF whose SAGE_HOTSPOT description does not match constants.py.""" + with open(path, 'w') as fh: + fh.write( + '##fileformat=VCFv4.2\n' + '##FILTER=\n' + '##INFO=\n' + '##contig=\n' + '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n' + 'chr1\t100\t.\tA\tT\t.\tPASS\tSAGE_HOTSPOT\n' + ) + + +class TestAnnotateExistingSageCallsHeaderCheck(unittest.TestCase): + """Regression test for rescue.py's SAGE VCF header-consistency check. + + annotate_existing_sage_calls(input_fp, tumor_name, sage_vcf_fp, output_dir) + must validate header descriptions against sage_vcf_fp (the SAGE VCF), not + input_fp (the DRAGEN VCF being annotated). Prior to the fix, the check was + wired to input_fp, which has no SAGE_* header lines at all — so a mismatch + in the real SAGE VCF's headers went undetected. + """ + + def test_header_mismatch_in_sage_vcf_triggers_exit(self): + """A mismatched SAGE_HOTSPOT description in sage_vcf_fp must raise SystemExit. + + input_fp has no SAGE_* headers at all, so if the check were (incorrectly) + run against input_fp instead, every field would be silently skipped and + no SystemExit would be raised — proving the check now reads sage_vcf_fp. + """ + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + input_fp = tmp_path / 'input.vcf' + sage_vcf_fp = tmp_path / 'sage.vcf' + _write_input_vcf_no_sage_fields(input_fp) + _write_sage_vcf_mismatched_hotspot_description(sage_vcf_fp) + + with self.assertRaises(SystemExit): + rescue.annotate_existing_sage_calls( + input_fp, 'TUMOR', sage_vcf_fp, tmp_path, + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_util.py b/tests/test_util.py index 7cac978..bdd4ab0 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -184,6 +184,60 @@ def test_output_is_gzipped(self): self.assertEqual(magic, b'\x1f\x8b') +class TestCheckAnnotationHeaders(unittest.TestCase): + """Unit tests for util.check_annotation_headers().""" + + def _write_vcf_with_sage_hotspot(self, path, description): + with open(path, 'w') as fh: + fh.write( + '##fileformat=VCFv4.2\n' + '##FILTER=\n' + f'##INFO=\n' + '##contig=\n' + '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n' + 'chr1\t100\t.\tA\tT\t.\tPASS\t.\n' + ) + + def _write_vcf_without_sage_fields(self, path): + with open(path, 'w') as fh: + fh.write( + '##fileformat=VCFv4.2\n' + '##FILTER=\n' + '##contig=\n' + '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n' + 'chr1\t100\t.\tA\tT\t.\tPASS\t.\n' + ) + + def test_matching_description_returns_normally(self): + expected = constants.VCF_HEADER_ENTRIES[constants.VcfInfo.SAGE_HOTSPOT]['Description'] + with tempfile.TemporaryDirectory() as tmp: + vcf_fp = pathlib.Path(tmp) / 'matching.vcf' + self._write_vcf_with_sage_hotspot(vcf_fp, expected) + # No exception/SystemExit raised + util.check_annotation_headers( + {constants.VcfInfo.SAGE_HOTSPOT: 'SAGE_HOTSPOT'}, vcf_fp, + ) + + def test_mismatched_description_exits(self): + with tempfile.TemporaryDirectory() as tmp: + vcf_fp = pathlib.Path(tmp) / 'mismatched.vcf' + self._write_vcf_with_sage_hotspot(vcf_fp, 'a totally different description') + with self.assertRaises(SystemExit): + util.check_annotation_headers( + {constants.VcfInfo.SAGE_HOTSPOT: 'SAGE_HOTSPOT'}, vcf_fp, + ) + + def test_field_absent_from_target_vcf_is_skipped(self): + with tempfile.TemporaryDirectory() as tmp: + vcf_fp = pathlib.Path(tmp) / 'no_sage.vcf' + self._write_vcf_without_sage_fields(vcf_fp) + # SAGE_HOTSPOT has no header entry in this VCF at all; must be + # skipped rather than raising, so no exception/SystemExit here. + util.check_annotation_headers( + {constants.VcfInfo.SAGE_HOTSPOT: 'SAGE_HOTSPOT'}, vcf_fp, + ) + + @unittest.skipUnless(shutil.which('bcftools'), 'bcftools not available') class TestMergeVcfFiles(unittest.TestCase): """Integration tests for util.merge_vcf_files(). From 2456ab715f67141b536cc179ede6258d576ca161 Mon Sep 17 00:00:00 2001 From: Quentin Clayssen <37511834+qclayssen@users.noreply.github.com> Date: Wed, 22 Jul 2026 10:47:25 +1000 Subject: [PATCH 89/89] fix: skip bcftools merge for single PCGR VCF chunk (#26) (#36) run_somatic_chunk -> merging_pcgr_files unconditionally called util.merge_vcf_files, which shells out to `bcftools merge`. That command requires 2+ inputs, so a sample whose variants fit in a single chunk (no splitting needed) produced a bcftools merge Usage error instead of a valid annotated VCF. Guard on len(pcgr_vcf_files) == 1: copy the single chunk's VCF directly to the expected merged-output path and tabix-index it, matching the indexed .vcf.gz contract that merge_vcf_files normally produces. Two or more chunks still go through the existing bcftools merge -> sort -> index path unchanged. --- bolt/common/pcgr.py | 9 ++++++- tests/test_pcgr.py | 60 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/bolt/common/pcgr.py b/bolt/common/pcgr.py index b8989cf..642dfb9 100644 --- a/bolt/common/pcgr.py +++ b/bolt/common/pcgr.py @@ -606,7 +606,14 @@ def merging_pcgr_files(output_dir, pcgr_vcf_files, pcgr_tsv_files): util.merge_tsv_files(pcgr_tsv_files, merged_tsv_fp) merged_vcf_path = pcgr_dir / "nosampleset.pcgr.grch38.pass" - merged_vcf = util.merge_vcf_files(pcgr_vcf_files, merged_vcf_path) + if len(pcgr_vcf_files) == 1: + # NOTE(QC): bcftools merge requires 2+ inputs; with a single chunk there is + # nothing to merge, so use that chunk directly as the merged output (bolt #26) + merged_vcf = merged_vcf_path.parent / f'{merged_vcf_path.name}.vcf.gz' + shutil.copy(pcgr_vcf_files[0], merged_vcf) + util.execute_command(f'bcftools index -t {merged_vcf}') + else: + merged_vcf = util.merge_vcf_files(pcgr_vcf_files, merged_vcf_path) return merged_vcf, merged_tsv_fp diff --git a/tests/test_pcgr.py b/tests/test_pcgr.py index 04759b4..418e1b3 100644 --- a/tests/test_pcgr.py +++ b/tests/test_pcgr.py @@ -1,4 +1,5 @@ """Tests for bolt/common/pcgr.py — tier ordering, filter categorisation, chunking.""" +import gzip import pathlib import tempfile import unittest @@ -6,6 +7,7 @@ import bolt.common.constants as constants import bolt.common.pcgr as pcgr +import bolt.util as util from tests.helpers import _csq, _count_vcf, _make_variant, _write_vcf @@ -298,6 +300,64 @@ def fake_run_somatic(*args, **kwargs): ) +class TestMergingPcgrFiles(unittest.TestCase): + """Regression test for bolt #26: bcftools merge requires 2+ inputs. + + When a sample's variants fit in a single PCGR chunk, run_somatic_chunk still + called merging_pcgr_files() -> util.merge_vcf_files() unconditionally, which + invoked `bcftools merge` on a single VCF and errored (Usage: bcftools merge + [options] [...]). + """ + + def _write_gz_vcf(self, path, variants): + vcf_path = path.with_suffix('') + _write_vcf(vcf_path, variants) + util.execute_command(f'bcftools view -Oz -o {path} {vcf_path}') + util.execute_command(f'bcftools index -t {path}') + + def test_single_chunk_skips_bcftools_merge(self): + """A single VCF chunk must bypass bcftools merge and pass through directly.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + vcf_fp = tmp_path / 'chunk1.vcf.gz' + self._write_gz_vcf(vcf_fp, [(10, f'PCGR_CSQ={_csq("intron_variant")}')]) + + tsv_fp = tmp_path / 'chunk1.tsv.gz' + with gzip.open(tsv_fp, 'wt') as fh: + fh.write('col1\tcol2\nval1\tval2\n') + + merged_vcf, merged_tsv = pcgr.merging_pcgr_files(tmp_path, [vcf_fp], [tsv_fp]) + + self.assertTrue(pathlib.Path(merged_vcf).exists()) + self.assertEqual(_count_vcf(merged_vcf), 1) + self.assertTrue(pathlib.Path(f'{merged_vcf}.tbi').exists(), + 'Single-chunk pass-through VCF must still be tabix indexed') + self.assertTrue(pathlib.Path(merged_tsv).exists()) + + def test_multiple_chunks_still_merge(self): + """Two or more chunks must still go through bcftools merge as before.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + vcf1_fp = tmp_path / 'chunk1.vcf.gz' + vcf2_fp = tmp_path / 'chunk2.vcf.gz' + self._write_gz_vcf(vcf1_fp, [(10, f'PCGR_CSQ={_csq("intron_variant")}')]) + self._write_gz_vcf(vcf2_fp, [(20, f'PCGR_CSQ={_csq("intron_variant")}')]) + + tsv1_fp = tmp_path / 'chunk1.tsv.gz' + tsv2_fp = tmp_path / 'chunk2.tsv.gz' + with gzip.open(tsv1_fp, 'wt') as fh: + fh.write('col1\tcol2\nval1\tval2\n') + with gzip.open(tsv2_fp, 'wt') as fh: + fh.write('col1\tcol2\nval3\tval4\n') + + merged_vcf, merged_tsv = pcgr.merging_pcgr_files( + tmp_path, [vcf1_fp, vcf2_fp], [tsv1_fp, tsv2_fp] + ) + + self.assertTrue(pathlib.Path(merged_vcf).exists()) + self.assertTrue(pathlib.Path(merged_tsv).exists()) + + class TestRunSomaticCommandArgs(unittest.TestCase): """Verify run_somatic builds the correct PCGR command-line arguments."""