diff --git a/arc/job/trsh.py b/arc/job/trsh.py index f1878a7011..222cecbba0 100644 --- a/arc/job/trsh.py +++ b/arc/job/trsh.py @@ -1137,10 +1137,22 @@ def trsh_ess_job(label: str, couldnt_trsh = True if couldnt_trsh: - logger.error(f'Could not troubleshoot geometry optimization for {label}! ' - f'Tried troubleshooting with the following methods: {ess_trsh_methods}') + # Count and remove 'trsh_attempt' entries for cleaner reporting + trsh_attempt_count = ess_trsh_methods.count('trsh_attempt') + filtered_methods = [method for method in ess_trsh_methods if method != 'trsh_attempt'] + + # Build the message with the count and filtered methods + if trsh_attempt_count > 0 and filtered_methods: + message = f'Tried troubleshooting {trsh_attempt_count} times, with the following methods: {filtered_methods}' + elif trsh_attempt_count > 0: + message = f'Tried troubleshooting {trsh_attempt_count} times' + else: + message = f'Tried troubleshooting with the following methods: {filtered_methods}' + + logger.error(f'Could not troubleshoot {job_type} for {label}! ' + f'{message}') output_errors.append(f'Error: Could not troubleshoot {job_type} for {label}! ' - f'Tried troubleshooting with the following methods: {ess_trsh_methods}; ') + f'{message}; ') return (output_errors, ess_trsh_methods, remove_checkfile, diff --git a/arc/job/trsh_test.py b/arc/job/trsh_test.py index d974874e9c..0802db5195 100644 --- a/arc/job/trsh_test.py +++ b/arc/job/trsh_test.py @@ -445,6 +445,27 @@ def test_trsh_ess_job(self): output_errors, ) + # Gaussian: test 6b - verify troubleshoot attempts counting + job_status = {'keywords': ['MaxOptCycles', 'GL9999']} + ess_trsh_methods = ['trsh_attempt', + 'int=(Acc2E=14)', 'opt=(maxcycle=200)', + 'trsh_attempt', 'opt=(RFO)', + 'trsh_attempt', 'opt=(GDIIS)', + 'trsh_attempt', 'opt=(GEDIIS)', + 'trsh_attempt'] + output_errors, ess_trsh_methods, remove_checkfile, level_of_theory, software, job_type, fine, trsh_keyword, \ + memory, shift, cpu_cores, couldnt_trsh = trsh.trsh_ess_job(label, level_of_theory, server, job_status, + job_type, software, fine, memory_gb, + num_heavy_atoms, cpu_cores, ess_trsh_methods) + self.assertTrue(couldnt_trsh) + self.assertIn('Tried troubleshooting 5 times, with the following methods:', output_errors[-1]) + self.assertNotIn('trsh_attempt', output_errors[-1]) + self.assertIn("opt=(maxcycle=200)", output_errors[-1]) + self.assertIn("opt=(RFO)", output_errors[-1]) + self.assertIn("opt=(GDIIS)", output_errors[-1]) + self.assertIn("opt=(GEDIIS)", output_errors[-1]) + self.assertIn('all_attempted', output_errors[-1]) + # Gaussian: test 7 job_status = {'keywords': ['MaxOptCycles', 'GL9999','SCF']} ess_trsh_methods = ['int=(Acc2E=14)'] diff --git a/arc/scheduler.py b/arc/scheduler.py index c56fae7d72..04107f04f9 100644 --- a/arc/scheduler.py +++ b/arc/scheduler.py @@ -3554,7 +3554,8 @@ def troubleshoot_ess(self, level_of_theory = Level(repr=level_of_theory) logger.info('\n') - warning_message = f'Troubleshooting {label} job {job.job_name} which failed' + # log job failure information before troubleshooting + warning_message = f'{label} Job {job.job_name} failed' if job.job_status[1]["status"] and job.job_status[1]["status"] != 'done': warning_message += f' with status: "{job.job_status[1]["status"]},"' if job.job_status[1]["keywords"]: @@ -3593,6 +3594,8 @@ def troubleshoot_ess(self, f'Reached max troubleshooting attempts ({max_ess_trsh}).') self.output[label]['errors'] += f'Error: ESS troubleshooting attempts exhausted for {label} {job.job_type}; ' return + logger.warning(f'Troubleshooting {label} job {job.job_name} ' + f'(attempt number {trsh_attempts + 1}).') job.ess_trsh_methods.append('trsh_attempt') # Determine if the species is a hydrogen atom (or its isotope).