diff --git a/comparisons/moduleAllocMonitor-circles-diff.py b/comparisons/moduleAllocMonitor-circles-diff.py index 621842ca03c..1c502c99032 100755 --- a/comparisons/moduleAllocMonitor-circles-diff.py +++ b/comparisons/moduleAllocMonitor-circles-diff.py @@ -4,8 +4,28 @@ import json import os -threshold = 5000.0 -error_threshold = 20000.0 + +def build_viewer_html(template_path, embedded_data, source_label): + with open(template_path, encoding="utf-8") as template_file: + content = template_file.read() + + embedded_json = json.dumps(embedded_data).replace("", "<\\/") + + autoload_snippet = """ + +""" % (embedded_json, json.dumps(source_label)) + + if "
| ', - "warn threshold %0.2f kB" % threshold, - ' |
| ', - "error threshold %0.2f kB" % error_threshold, - ' |
| ', - "warn threshold -%0.2f kB" % threshold, - ' |
| ', - "warn threshold -%0.2f kB" % error_threshold, - " |
| metric: <baseline> <pull request> <PR - baseline> | ",
- "
| Type Label | ',
- ]
- summary_header += build_header_row()
- summary_header += [
- "
| %s %s | " % (prdata["total"]["type"], prdata["total"]["label"]),
- ]
- for metric in METRICS_KEYS:
- append_triplet_cell(
- summary_header,
- sum_numeric_values(
- results["total"], ["%s %s IB" % (metric, key) for key in BEGIN_JOB_KEYS]
- ),
- sum_numeric_values(
- results["total"], ["%s %s PR" % (metric, key) for key in BEGIN_JOB_KEYS]
- ),
- sum_numeric_values(
- results["total"], ["%s %s diff" % (metric, key) for key in BEGIN_JOB_KEYS]
- ),
- )
- append_triplet_cell(
- summary_header,
- sum_numeric_values(
- results["total"], ["%s %s IB" % (metric, key) for key in CONSTRUCTION_KEYS]
- ),
- sum_numeric_values(
- results["total"], ["%s %s PR" % (metric, key) for key in CONSTRUCTION_KEYS]
- ),
- sum_numeric_values(
- results["total"], ["%s %s diff" % (metric, key) for key in CONSTRUCTION_KEYS]
- ),
- )
- append_triplet_cell(
- summary_header,
- sum_numeric_values(
- results["total"], ["%s %s IB" % (metric, key) for key in BEGIN_RUN_KEYS]
- ),
- sum_numeric_values(
- results["total"], ["%s %s PR" % (metric, key) for key in BEGIN_RUN_KEYS]
- ),
- sum_numeric_values(
- results["total"], ["%s %s diff" % (metric, key) for key in BEGIN_RUN_KEYS]
- ),
- )
- append_triplet_cell(
- summary_header,
- sum_numeric_values(
- results["total"], ["%s %s IB" % (metric, key) for key in BEGIN_LUMI_KEYS]
- ),
- sum_numeric_values(
- results["total"], ["%s %s PR" % (metric, key) for key in BEGIN_LUMI_KEYS]
- ),
- sum_numeric_values(
- results["total"], ["%s %s diff" % (metric, key) for key in BEGIN_LUMI_KEYS]
- ),
- )
- append_triplet_cell(
- summary_header,
- sum_numeric_values(
- results["total"], ["%s %s IB" % (metric, key) for key in EVENT_KEYS]
- ),
- sum_numeric_values(
- results["total"], ["%s %s PR" % (metric, key) for key in EVENT_KEYS]
- ),
- sum_numeric_values(
- results["total"], ["%s %s diff" % (metric, key) for key in EVENT_KEYS]
- ),
- )
- append_triplet_cell(
- summary_header,
- sum_numeric_values(
- results["total"], ["%s %s IB" % (metric, key) for key in EVENT_SETUP_KEYS]
- ),
- sum_numeric_values(
- results["total"], ["%s %s PR" % (metric, key) for key in EVENT_SETUP_KEYS]
- ),
- sum_numeric_values(
- results["total"], ["%s %s diff" % (metric, key) for key in EVENT_SETUP_KEYS]
- ),
- )
- append_triplet_cell(
- summary_header,
- sum_numeric_values(
- results["total"], ["%s %s IB" % (metric, key) for key in TOTAL_KEYS]
- ),
- sum_numeric_values(
- results["total"], ["%s %s PR" % (metric, key) for key in TOTAL_KEYS]
- ),
- sum_numeric_values(
- results["total"], ["%s %s diff" % (metric, key) for key in TOTAL_KEYS]
- ),
- )
- summary_header += [
- "
| Module label Module type Module record | ',
- ]
- summary_header += build_header_row()
- summary_header += [
- 'transitions | ', - "
| %s %s %s | '
- % (moduleres.get("label", ""), moduleres.get("type", ""), moduleres.get("record", "")),
- ]
- for metric in METRICS_KEYS:
- append_module_columns_prefix(summary_lines, moduleres, metric)
- transitions_ib = numeric_value(moduleib, "transitions")
- transitions_pr = numeric_value(modulepr, "transitions")
- transitions_diff = transitions_diff_value(transitions_ib, transitions_pr)
- append_triplet_cell(summary_lines, transitions_ib, transitions_pr, transitions_diff)
- summary_lines += ["
" in content: + content = content.replace("", autoload_snippet + "", 1) + return content + BEGIN_JOB_KEYS = ["begin job"] BEGIN_RUN_KEYS = ["global begin run", "stream begin run"] @@ -28,62 +48,69 @@ def module_key(module): - return "%s|%s|%s" % (module.get("label", ""), module.get("type", ""), module.get("record", "")) + return "%s|%s" % (module.get("label", ""), module.get("type", "")) -def numeric_value(data, key, default="N/A"): - value = data.get(key, default) - return value if isinstance(value, (int, float)) else default +def module_record(module): + record = module.get("record", "") + return str(record).strip() -def sum_numeric_values(data, keys, default="N/A"): - values = [data.get(key, default) for key in keys] - return sum(values) if all(isinstance(value, (int, float)) for value in values) else default +def is_event_setup_metric(metric): + return str(metric).strip().lower().endswith(" event setup") -def sum_with_prefix_suffix(data, metric_keys, prefix="added", suffix="", default="N/A"): - return sum_numeric_values( - data, ["%s %s %s" % (prefix, metric, suffix) for metric in metric_keys], default - ) +def to_numeric_or_none(value): + if isinstance(value, (int, float)): + return value + if isinstance(value, dict): + total_value = value.get("total") + if isinstance(total_value, (int, float)): + return total_value + values = [sub_value for sub_value in value.values() if isinstance(sub_value, (int, float))] + return sum(values) if values else None + return None -def format_metric(value): - return f"{value:.2f}" if isinstance(value, float) else str(value) +def sum_numeric_values(data, keys, default="N/A"): + values = [to_numeric_or_none(data.get(key, default)) for key in keys] + return sum(values) if all(value is not None for value in values) else default -def append_triplet_cell(summary_lines, ib, pr, diff, attrs='align="right"'): - summary_lines.append( - "
" - % (attrs, format_metric(ib), format_metric(pr), format_metric(diff)) +def sum_with_prefix_suffix(data, metric_keys, prefix="added", suffix="", default="N/A"): + return sum_numeric_values( + data, ["%s %s %s" % (prefix, metric, suffix) for metric in metric_keys], default ) -def added_total_color(diff_value): - if not isinstance(diff_value, (int, float)): - return "" - if diff_value > error_threshold: - return 'bgcolor="red"' - if diff_value > threshold: - return 'bgcolor="orange"' - if diff_value < -1.0 * error_threshold: - return 'bgcolor="green"' - if diff_value < -1.0 * threshold: - return 'bgcolor="cyan"' - return "" - - def is_valid_module_key(key): return key != "None|None|None" and key != "||" -def transitions_diff_value(transitions_ib, transitions_pr): - if isinstance(transitions_ib, (int, float)) and isinstance(transitions_pr, (int, float)): - return transitions_ib - transitions_pr - if not isinstance(transitions_ib, (int, float)) and isinstance(transitions_pr, (int, float)): - return transitions_pr - 0 - if isinstance(transitions_ib, (int, float)) and not isinstance(transitions_pr, (int, float)): - return 0 - transitions_ib - return "N/A" +def compute_event_setup_total(event_setup): + total = 0 + if not isinstance(event_setup, dict): + return total + for key, value in event_setup.items(): + if key == "total": + continue + if isinstance(value, (int, float)): + total += value + elif isinstance(value, dict): + for nested in value.values(): + if isinstance(nested, (int, float)): + total += nested + return total + + +def ensure_event_setup_total(module): + for key in ["IB", "PR", "diff"]: + for metric in METRICS_KEYS: + event_setup = module.get("%s event setup %s" % (metric, key)) + if not isinstance(event_setup, dict): + event_setup = {"total": 0} + event_setup["total"] = compute_event_setup_total(event_setup) + module["%s event setup %s" % (metric, key)] = event_setup def update_added_totals(datamapres): @@ -103,267 +130,70 @@ def update_added_totals(datamapres): ) -def build_header_row(): - return [ - '
', - '
', - '
', - '
', - '
', - '
', - '
', - '
', - '
', - '
', - '
', - '
', - '
', - '
', - '
', - '
', - '
', - '
', - '
', - '
', - '
', - '
', - '
', - '
', - '
', - '
', - '
', - '
', - '
', - '
', - '
', - '
', - '
', - '
', - '
', - ] - - -def build_summary_header(ibdata, prdata, results): - summary_header = [ - "", - "
", - "", - "