From 44e8aa8870e3b1b88d6408802269a12f1d6fdf74 Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Fri, 12 Dec 2025 11:53:22 -0600 Subject: [PATCH] Squashed commit of the following: commit fc3f8e209bd2e19230e1c5509cfc103903509c76 Author: Patrick Gartung Date: Fri Dec 12 11:51:45 2025 -0600 Make warning and error messages shorter commit 25ca5619ca213ed598ac2eef9fd7b6415f756d97 Author: Patrick Gartung Date: Fri Dec 12 08:20:11 2025 -0600 Take out 
 from list output

commit 3783bbb5578613a6c9e2eaeccaadfa936daaab0f
Author: Patrick Gartung 
Date:   Thu Dec 11 18:07:05 2025 -0600

    Markdown formatting for list of workflow failing threshold

commit 9281a5108210411521cf998df5a873e19264fbe7
Author: Patrick Gartung 
Date:   Thu Dec 11 15:09:40 2025 -0600

    Change the wording of failure messages

commit 0a30ff472417f7178e11efc939f74572374a4509
Author: Patrick Gartung 
Date:   Thu Dec 11 13:28:58 2025 -0600

    Split the grep output and report just the match string

commit fcac2ad0b55f0bc79796dffa3354dccd374673b3
Author: Patrick Gartung 
Date:   Thu Dec 11 12:47:34 2025 -0600

    Remove --repo option with empty var as arg

commit bbe95388d98a85808afec0e95c985cd772558b58
Author: Patrick Gartung 
Date:   Thu Dec 11 11:28:21 2025 -0600

    Echo report-pull-request-results command

commit 419951a7d9c0a2e508f4d223b0c79963fe6bf244
Author: Patrick Gartung 
Date:   Thu Dec 11 10:24:20 2025 -0600

    Make sure testResults directory is created before writing maxmem files

commit 6dda6b2780563e8a96a574ff55d362414a9a830d
Author: Patrick Gartung 
Date:   Wed Dec 10 17:40:38 2025 -0600

    Fix read_maxmem_comparison_file

commit 59dbbb25ff9405b8e427f8445278b2e00a51670b
Author: Patrick Gartung 
Date:   Wed Dec 10 17:35:10 2025 -0600

    Process the maxmem_comparison.log file instead of the individual .err file

commit 70343d58734f5d391c69d42b677ec065826a72bd
Author: Patrick Gartung 
Date:   Wed Dec 10 17:24:03 2025 -0600

    Missing space

commit 48e99bde4c9d44f91ecfaed2be8fe3f1f14931e8
Author: Patrick Gartung 
Date:   Wed Dec 10 17:12:02 2025 -0600

    Log output of grep to maxmem_summary.log

commit 78ba103a2cd7a0b138bf33f86ddce3cad136b0c4
Author: Patrick Gartung 
Date:   Wed Dec 10 14:25:56 2025 -0600

    Send the output of maxmem comparison to log

commit 0253ccaad922658732c1f0404023b7952f17a2c7
Author: Patrick Gartung 
Date:   Wed Dec 10 11:39:17 2025 -0600

    Add --no-post report-pull-request-results so it prints to the console to see if send_nessage_pr is called for maxmem

commit 62ab359f1f89d7144856b346d2fa7fe6e8caad3b
Author: Patrick Gartung 
Date:   Tue Dec 9 16:41:22 2025 -0600

    COMMIT_SHA not defined

commit dc115476b81a11ead59d44dd69920aeaedf7a941
Author: Patrick Gartung 
Date:   Mon Dec 8 13:51:10 2025 -0600

    Fix report-pull-request-result invocation

commit fbf497b42af0d3896af1fd815d05a17fb0dbc7a9
Author: Patrick Gartung 
Date:   Mon Dec 8 11:37:32 2025 -0600

    Change max-memory-used check to use real difference in MB not percentage diff. Increase the threshold to 80MB. Flag PR is check fails.

    Squashed commit of the following:

    commit b9f2dc1dbca3bcae799a47aafe25fcf1fc3494e3
    Author: Patrick Gartung 
    Date:   Mon Dec 8 11:35:14 2025 -0600

        Use yellow and green to indicate negative warn and error thresholds

    commit fe76627a8405bebf4d279f757dbcd89174cde644
    Author: Patrick Gartung 
    Date:   Mon Dec 8 11:12:30 2025 -0600

        Use blud and green to indicate negative warn and error thresholds

    commit 2cbb34ac362a2814a32f230cc093014bc6d6939d
    Author: Patrick Gartung 
    Date:   Fri Dec 5 13:27:06 2025 -0600

        black formatting

    commit cd466d9883141f4a6aacfb9a085a1c62c1d2b058
    Author: Patrick Gartung 
    Date:   Fri Dec 5 13:24:38 2025 -0600

        Use MiB scaling 1024*1024

    commit 79d93d4e62abc5107f09e8736819e99e206cfd18
    Author: Patrick Gartung 
    Date:   Fri Dec 5 12:43:36 2025 -0600

        Add missing \n

    commit 1ed1123e11e88864668c3f6899f9b59f4e0b8d9d
    Author: Patrick Gartung 
    Date:   Fri Dec 5 12:40:15 2025 -0600

        Use MiB not MB. Make sys.stderr.write append instead of overwriting

    commit 4f894aaedf0999feba56bcdbff0deac819c9caa6
    Author: Patrick Gartung 
    Date:   Fri Dec 5 12:21:42 2025 -0600

        get_result_file_name should handle maxmem

    commit 998aa34f3decf0404ec76fd01782fa64627eadd0
    Author: gartung 
    Date:   Thu Dec 4 16:49:29 2025 -0600

        More black formatting

    commit e1f45ebf067ba305296183003958e2de48d55b1a
    Author: gartung 
    Date:   Thu Dec 4 16:46:01 2025 -0600

        black formatting

    commit 888be7288479c0e6131e2b2619c1a84689d8982a
    Author: gartung 
    Date:   Thu Dec 4 16:43:06 2025 -0600

        Change message in summary table to say default threshold values. Remove unneeded path insertion.

    commit cc5332d8e498a2ecca57b195d4856c126511fd47
    Author: gartung 
    Date:   Thu Dec 4 10:49:38 2025 -0600

        More black formatting

    commit ae2568022d074596e280dd9eeb6b5bd59c47e074
    Author: gartung 
    Date:   Thu Dec 4 10:47:14 2025 -0600

        black formatting

    commit 4f8aeec9a2e166f0aac6bf1418254323a4d08190
    Author: gartung 
    Date:   Thu Dec 4 10:43:42 2025 -0600

        Make error threshold 80MB

    commit bd3977b76368e4d3bfccccd6bbc395ae18dd4583
    Author: gartung 
    Date:   Thu Dec 4 10:40:58 2025 -0600

        Add script directory to PYTHONPATH so maxmem_threshold.py can be found. Use consistent units of (1024*1024) bytes per MB.

    commit b8413cfa71428d0513906162ee88b3b5d69317bd
    Author: gartung 
    Date:   Thu Dec 4 10:21:58 2025 -0600

        Put thresholds in their own file imported by the scripts that use them.

    commit 27c4e28fe3692900e09924d15fdb875baaf93a6a
    Author: gartung 
    Date:   Wed Dec 3 15:47:04 2025 -0600

        Send message is exceeds error threshold appears in *.err file

    commit 17bd472f3e9c55f6da31e9cb1dc92f54fe6f9d1e
    Author: gartung 
    Date:   Wed Dec 3 15:45:31 2025 -0600

        Send message is exceeds appear in *.err file

    commit 83ff660d1a787b7e75f43164483d99e505cae331
    Author: gartung 
    Date:   Wed Dec 3 13:05:29 2025 -0600

        Add units to err message

    commit 340b386732a30a0ca1745be6cbfe30f8024a3e18
    Author: gartung 
    Date:   Wed Dec 3 12:54:35 2025 -0600

        Link to maxmem_summary.html

    commit 9964fb0f24471a3d181c8de84262ce0c62869900
    Author: gartung 
    Date:   Wed Dec 3 12:47:29 2025 -0600

        Add @cms-sw/core2 to message sent to PR

    commit ad79813c2ce9f306d745c393cda9c09f5c8d1402
    Author: gartung 
    Date:   Wed Dec 3 11:28:22 2025 -0600

        Fix error message

    commit aec4220fb78ba221a921207f800e8f8dcc25c69c
    Author: gartung 
    Date:   Wed Dec 3 11:23:27 2025 -0600

        Fix indent problem

    commit a2f4385e241994e2bd23209c4b0296192bdaacb4
    Author: gartung 
    Date:   Wed Dec 3 11:20:07 2025 -0600

        Scale max memory used diff before applying threshold.

    commit cce8d1e8462ed6a6f33d037bf0d2ec6a1d5cf895
    Author: gartung 
    Date:   Wed Dec 3 11:19:29 2025 -0600

        Fix table borders

    commit 1643523f7fbe985fc8e1ac007a63b984d2b6f72a
    Author: Patrick Gartung 
    Date:   Wed Dec 3 07:28:46 2025 -0600

        Put % in correct location. Rename index.html maxmem_summary.html

    commit 790acfa7d0bb21818354efc06671ed5320da7cd2
    Author: Patrick Gartung 
    Date:   Tue Dec 2 16:58:26 2025 -0600

        Add defintion of mem_prof_pdiffs_dicts

    commit 157a4408051850ca3f45808a77a94033bf9143b4
    Author: Patrick Gartung 
    Date:   Tue Dec 2 16:51:14 2025 -0600

        Black formatting

    commit 94b40ddaf5993d6371c651254494991c2b418be5
    Author: Patrick Gartung 
    Date:   Tue Dec 2 16:47:01 2025 -0600

        Add changes suggested by CoPilot to report which workflows exceed threshold.

    commit 82d77ee35e0e49eb20aa2e3dbd4446d896a5303b
    Merge: 901627be f270b07e
    Author: Patrick Gartung 
    Date:   Tue Dec 2 14:53:44 2025 -0600

        Merge remote-tracking branch 'origin/master'

    commit 901627be525e9a3610ebbd8fb7fd20000c04fa81
    Author: Patrick Gartung 
    Date:   Tue Dec 2 14:52:30 2025 -0600

        Remove FAILED_ from MAXMEM_COMPARISON

    commit e27117c0b8e31e10ceed0ca327b04e0f98688903
    Author: Patrick Gartung 
    Date:   Tue Dec 2 14:51:24 2025 -0600

        Fix type in compare-maxmem.py

    commit db380dc48a6af98e8e24da9058d3309ee749df73
    Merge: bbca0ebe 8335960c
    Author: Patrick Gartung 
    Date:   Tue Dec 2 12:00:48 2025 -0600

        Merge remote-tracking branch 'refs/remotes/upstream/master'

    commit f270b07ee141d13d61b9282fcd8e6e3e4111e5da
    Merge: bbca0ebe 8335960c
    Author: Patrick Gartung 
    Date:   Tue Dec 2 11:56:53 2025 -0600

        Merge branch 'cms-sw:master' into master

    commit bbca0ebe74de66033f5f8be6bb4cfbc4df615601
    Author: Patrick Gartung 
    Date:   Fri Nov 21 12:57:15 2025 -0600

        Report diff not abs(diff)

    commit fa38d771d5b9c31919176c645cc6a3950d0051b2
    Author: Patrick Gartung 
    Date:   Fri Nov 21 12:55:18 2025 -0600

        Change comparison to real value diff not percentage diff

    commit 4bc2b7ee2fb20759e889f442b549ff1db5832d6f
    Author: Patrick Gartung 
    Date:   Fri Nov 21 12:43:19 2025 -0600

        Black formatting

    commit 725349d2feb5a0cd1760fdd58ccac25c8a1ef165
    Author: Patrick Gartung 
    Date:   Fri Nov 21 12:40:50 2025 -0600

        Fix another indent

    commit d7bd165f476935037ffa5f425a5ae27d63cc663a
    Author: Patrick Gartung 
    Date:   Fri Nov 21 12:39:25 2025 -0600

        Fix another indent

    commit e7e0b40739a6455fb465c5cf02e69ad05652ec08
    Author: Patrick Gartung 
    Date:   Fri Nov 21 12:37:33 2025 -0600

        Fix indent

    commit 0a5c9ea359161de9236381cdc8fdc9c7997968d9
    Merge: 249ddeb7 245504f1
    Author: Patrick Gartung 
    Date:   Fri Nov 21 12:30:18 2025 -0600

        Merge remote-tracking branch 'upstream'

    commit 249ddeb783603afb8673d77af1cecb39f8eaf2f7
    Author: Patrick Gartung 
    Date:   Fri Nov 21 12:24:21 2025 -0600

        Increase memory thresholds for warnings and errors in compare_maxmem_summary.py and use diff of real value not percentage value.

    commit 1ecd41724ce5a45c3d92261220eb97b66f4799a2
    Author: Patrick Gartung 
    Date:   Thu Nov 20 13:36:13 2025 -0600

        Increase the max memory difference threshold to 100MB. Add failed/FAILED to message to .
---
 comparisons/compare-maxmem-summary.py | 74 +++++++++++++++------------
 comparisons/compare-maxmem.py         | 32 ++++++++----
 comparisons/maxmem_threshold.py       |  2 +
 pr_testing/_helper_functions.sh       |  8 +++
 pr_testing/run-pr-comparisons         | 19 ++++---
 report-pull-request-results.py        | 24 ++++++++-
 6 files changed, 108 insertions(+), 51 deletions(-)
 create mode 100644 comparisons/maxmem_threshold.py

diff --git a/comparisons/compare-maxmem-summary.py b/comparisons/compare-maxmem-summary.py
index 60340bb40c1b..51583f4c08ad 100755
--- a/comparisons/compare-maxmem-summary.py
+++ b/comparisons/compare-maxmem-summary.py
@@ -8,9 +8,9 @@
 import json
 import glob
 import re
+import sys
 
-MAXMEM_WARN_THRESHOLD = 1.0
-MAXMEM_ERROR_THRESHOLD = 10.0
+import maxmem_threshold
 
 
 def KILL(message):
@@ -50,9 +50,9 @@ def compare_maxmem_summary(**kwargs):
                 nalloc_pr = max_memory_pr_dict[step].get("# allocations calls")
                 ndalloc_pr = max_memory_pr_dict[step].get("# deallocations calls")
                 nlalloc_pr = nalloc_pr - ndalloc_pr if (nalloc_pr and ndalloc_pr) else 0
-                max_memory_pr = max_mem_pr / 1000000 if max_mem_pr else 0.0
-                req_memory_pr = req_mem_pr / 1000000 if req_mem_pr else 0.0
-                leak_memory_pr = leak_mem_pr / 1000000 if leak_mem_pr else 0.0
+                max_memory_pr = max_mem_pr / (1024 * 1024) if max_mem_pr else 0.0
+                req_memory_pr = req_mem_pr / (1024 * 1024) if req_mem_pr else 0.0
+                leak_memory_pr = leak_mem_pr / (1024 * 1024) if leak_mem_pr else 0.0
                 nallocated_pr = nalloc_pr if nalloc_pr else 0
 
                 max_mem_base = max_memory_base_dict[step].get("max memory used")
@@ -61,9 +61,9 @@ def compare_maxmem_summary(**kwargs):
                 nalloc_base = max_memory_base_dict[step].get("# allocations calls")
                 ndalloc_base = max_memory_base_dict[step].get("# deallocations calls")
                 nlalloc_base = nalloc_base - ndalloc_base if (nalloc_base and ndalloc_base) else 0
-                max_memory_base = max_mem_base / 1000000 if max_mem_base else 0.0
-                req_memory_base = req_mem_base / 1000000 if req_mem_base else 0.0
-                leak_memory_base = leak_mem_base / 1000000 if leak_mem_base else 0.0
+                max_memory_base = max_mem_base / (1024 * 1024) if max_mem_base else 0.0
+                req_memory_base = req_mem_base / (1024 * 1024) if req_mem_base else 0.0
+                leak_memory_base = leak_mem_base / (1024 * 1024) if leak_mem_base else 0.0
                 nallocated_base = nalloc_base if nalloc_base else 0
 
                 max_mem_pdiff = max_memory_pdiff_dict[step].get("max memory used")
@@ -199,36 +199,40 @@ def stepfn(step):
         summaryLine += [
             '<PR - baseline (MB)>'
         ]
-        for step in sorted(workflows[workflow].keys(), key=stepfn):
-            summaryLine += [
-                '',
-                "{:,.2f}".format(workflows[workflow][step]["max memory adiff"]),
-                "",
-            ]
-        summaryLine += [
-            "",
-        ]
-        summaryLine += [
-            '<100 * (PR - baseline)/baseline >'
-        ]
         for step in sorted(workflows[workflow].keys(), key=stepfn):
             threshold = workflows[workflow][step]["threshold"]
             if not threshold:
-                threshold = 1.0
+                threshold = maxmem_threshold.WARN_THRESHOLD
             error_threshold = workflows[workflow][step].get("error_threshold")
             if not error_threshold:
-                error_threshold = 10.0
-            cellString = ' MAXMEM_WARN_THRESHOLD:
+            if workflows[workflow][step]["max memory adiff"] > threshold:
                 color = 'bgcolor="orange"'
-            if abs(workflows[workflow][step]["max memory pdiff"]) > MAXMEM_ERROR_THRESHOLD:
+            if workflows[workflow][step]["max memory adiff"] > error_threshold:
                 color = 'bgcolor="red"'
+            if workflows[workflow][step]["max memory adiff"] < -1 * threshold:
+                color = 'bgcolor="yellow"'
+            if workflows[workflow][step]["max memory adiff"] < -1 * error_threshold:
+                color = 'bgcolor="green"'
             cellString += color
             cellString += ">"
             summaryLine += [
                 cellString,
-                "{:,.3f}".format(workflows[workflow][step]["max memory pdiff"]),
+                "{:,.3f}".format(workflows[workflow][step]["max memory adiff"]),
+                "",
+            ]
+        summaryLine += [
+            "",
+        ]
+        summaryLine += [
+            '<100 * (PR - baseline)/baseline >'
+        ]
+        for step in sorted(workflows[workflow].keys(), key=stepfn):
+            summaryLine += [
+                '',
+                "{:,.2f}".format(workflows[workflow][step]["max memory pdiff"]),
                 "%",
             ]
         summaryLine += [
@@ -391,9 +395,6 @@ def stepfn(step):
                 "{:,}".format(workflows[workflow][step]["nallocated base"]),
                 "",
             ]
-        summaryLine += [
-            "",
-        ]
         summaryLine += [
             '<pull request >'
         ]
@@ -435,10 +436,17 @@ def stepfn(step):
     if summaryFormat == "html":
         summaryLines += [
             '",
+            + "default maximum memory used warn threshold %0.0f" % maxmem_threshold.WARN_THRESHOLD
+            + ' MB
'
-            + "maximum memory used warn threshold %0.3f" % MAXMEM_WARN_THRESHOLD
-            + '%
'
-            + "maximum memory used error threshold %0.3f" % MAXMEM_ERROR_THRESHOLD
-            + "%
'
+            + "default maximum memory used error threshold %0.0f"
+            % maxmem_threshold.ERROR_THRESHOLD
+            + ' MB
'
+            + "default maximum memory used warn threshold -1 * %0.0f"
+            % maxmem_threshold.WARN_THRESHOLD
+            + ' MB
'
+            + "default maximum memory used error threshold -1 * %0.0f"
+            % maxmem_threshold.ERROR_THRESHOLD
+            + " MB
",
         ]
         summaryLines += ["
"]
 
diff --git a/comparisons/compare-maxmem.py b/comparisons/compare-maxmem.py
index e7e704f19eb9..5e9b808a941e 100755
--- a/comparisons/compare-maxmem.py
+++ b/comparisons/compare-maxmem.py
@@ -1,8 +1,11 @@
 #!/usr/bin/env python3
+import os
 import sys
 import json
 from collections import defaultdict
 
+import maxmem_threshold
+
 
 def create_memory_report_dict(filename):
     memory_reports = dict(dict())
@@ -25,45 +28,54 @@ def create_memory_report_dict(filename):
 mem_prof_base_dicts = create_memory_report_dict(sys.argv[2])
 
 mem_prof_pdiffs_dicts = dict(dict())
+mem_prof_diffs_dicts = dict(dict())
 
 for k in mem_prof_pr_dicts.keys():
     mem_prof_pdiffs_dict = dict()
+    mem_prof_diffs_dict = dict()
     mem_prof_pr_subdict = mem_prof_pr_dicts[k]
     for j, v in mem_prof_pr_subdict.items():
         if j == "step":
             mem_prof_pdiffs_dict[j] = v
+            mem_prof_diffs_dict[j] = v
         else:
             mem_prof_pdiffs_dict[j] = (
                 100
                 * (mem_prof_pr_dicts[k][j] - mem_prof_base_dicts[k][j])
                 / mem_prof_base_dicts[k][j]
             )
+            mem_prof_diffs_dict[j] = mem_prof_pr_dicts[k][j] - mem_prof_base_dicts[k][j]
     mem_prof_pdiffs_dicts[k] = mem_prof_pdiffs_dict
+    mem_prof_diffs_dicts[k] = mem_prof_diffs_dict
 
 mem_prof = {}
 
 mem_prof["max memory pr"] = mem_prof_pr_dicts
 mem_prof["max memory base"] = mem_prof_base_dicts
 mem_prof["max memory pdiffs"] = mem_prof_pdiffs_dicts
-WARN_THRESHOLD = 1.0
-ERROR_THRESHOLD = 10.0
-mem_prof["threshold"] = WARN_THRESHOLD
-mem_prof["error_threshold"] = ERROR_THRESHOLD
+mem_prof["max memory diffs"] = mem_prof_diffs_dicts
+mem_prof["threshold"] = maxmem_threshold.WARN_THRESHOLD
+mem_prof["error_threshold"] = maxmem_threshold.ERROR_THRESHOLD
 mem_prof["workflow"] = sys.argv[1].split("/")[-2]
 sys.stdout.write(json.dumps(mem_prof))
 sys.stdout.write("\n")
 
 errs = 0
-for k in sorted(mem_prof_pdiffs_dicts.keys()):
-    mmu = mem_prof_pdiffs_dicts[k].get("max memory used")
+for k in sorted(mem_prof_diffs_dicts.keys()):
+    mmu = mem_prof_diffs_dicts[k].get("max memory used")
     if mmu:
-        if abs(mmu) > ERROR_THRESHOLD:
+        mmus = mmu / (1024 * 1024)
+        if mmus > maxmem_threshold.WARN_THRESHOLD or mmus < -1 * maxmem_threshold.WARN_THRESHOLD:
+            sys.stderr.write(
+                "Warning: Workflow %s %s max memory diff %.1f exceeds +/- %.1f MiB\n"
+                % (mem_prof["workflow"], k, mmus, maxmem_threshold.WARN_THRESHOLD)
+            )
+        if mmus > maxmem_threshold.ERROR_THRESHOLD or mmus < -1 * maxmem_threshold.ERROR_THRESHOLD:
             errs = errs + 1
             sys.stderr.write(
-                "Workflow %s %s max memory used percentage diff %2f%% exceeds error threshold %2f%%"
-                % (mem_prof["workflow"], k, abs(mmu), ERROR_THRESHOLD)
+                "Error: Workflow %s %s max memory diff %.1f exceeds +/- %.1f MiB\n"
+                % (mem_prof["workflow"], k, mmus, maxmem_threshold.ERROR_THRESHOLD)
             )
-            sys.stderr.write("\n")
 
 if errs > 0:
     exit(10)
diff --git a/comparisons/maxmem_threshold.py b/comparisons/maxmem_threshold.py
new file mode 100644
index 000000000000..a9fd77dbed87
--- /dev/null
+++ b/comparisons/maxmem_threshold.py
@@ -0,0 +1,2 @@
+WARN_THRESHOLD = 10.0
+ERROR_THRESHOLD = 80.0
diff --git a/pr_testing/_helper_functions.sh b/pr_testing/_helper_functions.sh
index bb44f8c507e6..ff740cd6ed8d 100755
--- a/pr_testing/_helper_functions.sh
+++ b/pr_testing/_helper_functions.sh
@@ -189,6 +189,14 @@ function get_result_file_name () {
       echo "21-${TEST_FLAVOR}-comparison-report.res"
       return 0
       ;;
+    maxmem)
+      if [ "$TEST_FLAVOR" != "" ]; then
+        echo "23-${TEST_FLAVOR}-maxmem-report.res"
+      else
+        echo "23-maxmem-report.res"
+      fi
+      return 0
+      ;;
   esac
   return 1
 }
diff --git a/pr_testing/run-pr-comparisons b/pr_testing/run-pr-comparisons
index fa3b29145393..ae2e8bcb992b 100755
--- a/pr_testing/run-pr-comparisons
+++ b/pr_testing/run-pr-comparisons
@@ -294,9 +294,9 @@ set +x
 # maxmem-profile comparison
 # --------------------------------------------------------------------------
 echo "Started maxmem-profile comparison at `date`"
-OUTPUT_DIR=$WORKSPACE/results/maxmem-comparison
+MAXMEM_COMPARISON_OUTPUT_DIR=$WORKSPACE/results/maxmem-comparison
 #create the output dir
-mkdir -p $OUTPUT_DIR
+mkdir -p $MAXMEM_COMPARISON_OUTPUT_DIR
 for maxmem in $(find $WORKSPACE/data/PR-${PR_NUM} -follow -name 'maxmem_profile_*.txt' -type f | sed "s|$WORKSPACE/data/PR-${PR_NUM}/||") ; do
   echo "Maxmem Profile> Working on ${maxmem}"
   if [ ! -e "$WORKSPACE/data/$COMPARISON_RELEASE/${maxmem}" ] ; then
@@ -305,13 +305,18 @@ for maxmem in $(find $WORKSPACE/data/PR-${PR_NUM} -follow -name 'maxmem_profile_
   fi
   WF_NUMBER=$(echo ${maxmem} | sed 's|_.*||')
   $CMS_BOT_DIR/comparisons/compare-maxmem.py $WORKSPACE/data/PR-${PR_NUM}/${maxmem} \
-    $WORKSPACE/data/$COMPARISON_RELEASE/${maxmem} > $OUTPUT_DIR/${WF_NUMBER}.json 2> $OUTPUT_DIR/${WF_NUMBER}.err || true
+    $WORKSPACE/data/$COMPARISON_RELEASE/${maxmem} > $MAXMEM_COMPARISON_OUTPUT_DIR/${WF_NUMBER}.json 2>> $MAXMEM_COMPARISON_OUTPUT_DIR/${WF_NUMBER}.err || true
 done
-$CMS_BOT_DIR/comparisons/compare-maxmem-summary.py -i $OUTPUT_DIR -f '*.json' -F html -o $OUTPUT_DIR/index.html -u $JENKINS_ARTIFACTS_URL/$PR_BASELINE_DIR || true
-if grep "exceeds threshold" $OUTPUT_DIR/*.err 2>/dev/null; then
-  echo "MAXMEM_COMPARISON${TEST_FLAVOR_STR};OK,max memory used ${UC_TEST_FLAVOR} comparison,See results,/SDT/jenkins-artifacts/$COMP_UPLOAD_DIR/maxmem-comparison" >> ${RESULTS_FILE}
+mkdir -p $WORKSPACE/testsResults
+$CMS_BOT_DIR/comparisons/compare-maxmem-summary.py -i $MAXMEM_COMPARISON_OUTPUT_DIR -f '*.json' -F html -o $MAXMEM_COMPARISON_OUTPUT_DIR/maxmem_summary.html -u $JENKINS_ARTIFACTS_URL/$PR_BASELINE_DIR >$MAXMEM_COMPARISON_OUTPUT_DIR/maxmem_summary.log 2>&1 || true
+if grep "Error:" $MAXMEM_COMPARISON_OUTPUT_DIR/*.err >$MAXMEM_COMPARISON_OUTPUT_DIR/maxmem_summary.log 2>/dev/null; then
+  echo "MAXMEM_COMPARISON${TEST_FLAVOR_STR};OK,${UC_TEST_FLAVOR} max memory used comparison failed,See failed results,/SDT/jenkins-artifacts/$COMP_UPLOAD_DIR/maxmem-comparison/maxmem_summary.html" >> ${RESULTS_FILE}
+  REPORT_FILE=$WORKSPACE/testsResults/$(get_result_file_name "maxmem" "${TEST_FLAVOR}" "")
+  touch $REPORT_FILE
+  ${CMS_BOT_DIR}/report-pull-request-results PARSE_MAXMEM_FAIL --no-post --unit-tests-file $MAXMEM_COMPARISON_OUTPUT_DIR/maxmem_summary.log --report-file ${REPORT_FILE} --report-url ${PR_RESULT_URL} || true
+  ${CMS_BOT_DIR}/report-pull-request-results PARSE_MAXMEM_FAIL --unit-tests-file $MAXMEM_COMPARISON_OUTPUT_DIR/maxmem_summary.log --report-file ${REPORT_FILE} --report-url ${PR_RESULT_URL} || true
 else
-  echo "MAXMEM_COMPARISON${TEST_FLAVOR_STR};OK,max memory used ${UC_TEST_FLAVOR} comparison,See results,/SDT/jenkins-artifacts/$COMP_UPLOAD_DIR/maxmem-comparison" >> ${RESULTS_FILE}
+  echo "MAXMEM_COMPARISON${TEST_FLAVOR_STR};OK,max memory used ${UC_TEST_FLAVOR} comparison,See results,/SDT/jenkins-artifacts/$COMP_UPLOAD_DIR/maxmem-comparison/maxmem_summary.html" >> ${RESULTS_FILE}
 fi
 
 # --------------------------------------------------------------------------
diff --git a/report-pull-request-results.py b/report-pull-request-results.py
index cddfe139a1ca..17dab45a3aaa 100755
--- a/report-pull-request-results.py
+++ b/report-pull-request-results.py
@@ -26,7 +26,7 @@
     usage="usage: %prog ACTION [options] \n ACTION = PARSE_UNIT_TESTS_FAIL | PARSE_BUILD_FAIL "
     "| PARSE_MATRIX_FAIL | COMPARISON_READY | GET_BASE_MESSAGE | PARSE_EXTERNAL_BUILD_FAIL "
     "| PARSE_ADDON_FAIL | PARSE_CRAB_FAIL | PARSE_CLANG_BUILD_FAIL | MATERIAL_BUDGET "
-    "| PYTHON3_FAIL | PARSE_GPU_UNIT_TESTS_FAIL | MERGE_COMMITS"
+    "| PYTHON3_FAIL | PARSE_GPU_UNIT_TESTS_FAIL | MERGE_COMMITS | PARSE_MAXMEM_FAIL "
 )
 
 parser.add_option(
@@ -294,6 +294,26 @@ def read_material_budget_log_file(unit_tests_file):
     send_message_pr(message)
 
 
+#
+# reads maxmem comparison error files
+#
+def read_maxmem_comparison_file(unit_tests_file):
+    errors_found = ""
+    err_cnt = 0
+    for line in openlog(unit_tests_file):
+        if "exceeds" in line.lower():
+            err_cnt += 1
+            errors_found += " - " + line.split(":")[1] + "\n"
+
+    if err_cnt > 0:
+        message = (
+            "\n## Max Memory Comparisons exceeding threshold\n\n"
+            "@cms-sw/core-l2 , I found %s workflow step(s) with memory usage exceeding the error threshold:\n\n%s"
+            % (err_cnt, errors_found)
+        )
+        send_message_pr(message)
+
+
 def get_recent_merges_message():
     message = ""
     if options.recent_merges_file:
@@ -643,6 +663,8 @@ def complain_missing_param(param_name):
     read_python3_file(options.unit_tests_file)
 elif ACTION == "MATERIAL_BUDGET":
     read_material_budget_log_file(options.unit_tests_file)
+elif ACTION == "PARSE_MAXMEM_FAIL":
+    read_maxmem_comparison_file(options.unit_tests_file)
 elif ACTION == "MERGE_COMMITS":
     add_to_report(get_recent_merges_message())
 elif ACTION == "PARSE_CUDA_UNIT_TESTS_FAIL":