From aeaef79d2557c66f9dddb989147a539a2d5b439e Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Wed, 3 May 2023 17:20:14 -0700 Subject: [PATCH 01/29] fix default worker bug with all steps --- CHANGELOG.md | 7 +++ merlin/spec/specification.py | 7 ++- tests/integration/test_definitions.py | 10 ++++ .../test_specs/default_worker_test.yaml | 56 +++++++++++++++++++ .../test_specs/no_default_worker_test.yaml | 56 +++++++++++++++++++ 5 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 tests/integration/test_specs/default_worker_test.yaml create mode 100644 tests/integration/test_specs/no_default_worker_test.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 42811b601..e5ba817ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] +### Fixed +- A bug where assigning a worker all steps also assigned steps to the default worker + +### Added +- Tests to make sure the default worker is being assigned properly + ## [1.10.0] ### Fixed - Pip wheel wasn't including .sh files for merlin examples diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 6fb524f5b..9f3a6c16f 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -385,8 +385,11 @@ def process_spec_defaults(self): MerlinSpec.fill_missing_defaults(worker_settings, defaults.WORKER) worker_steps.extend(worker_settings["steps"]) - # Figure out which steps still need workers - steps_that_need_workers = list(set(all_workflow_steps) - set(worker_steps)) + if "all" in worker_steps: + steps_that_need_workers = [] + else: + # Figure out which steps still need workers + steps_that_need_workers = list(set(all_workflow_steps) - set(worker_steps)) # If there are still steps remaining that haven't been assigned a worker yet, # assign the remaining steps to the default worker. If all the steps still need workers diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index 6fd8e3ce9..2b990ea40 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -283,6 +283,16 @@ def define_tests(): # pylint: disable=R0914,R0915 "conditions": [HasReturnCode(), HasRegex("custom_verify_queue")], "run type": "local", }, + "default_worker assigned": { + "cmds": f"{workers} {test_specs}/default_worker_test.yaml --echo", + "conditions": [HasReturnCode(), HasRegex(r"default_worker.*-Q '\[merlin\]_step_4_queue'")], + "run type": "local", + }, + "no default_worker assigned": { + "cmds": f"{workers} {test_specs}/no_default_worker_test.yaml --echo", + "conditions": [HasReturnCode(), HasRegex(r"default_worker", negate=True)], + "run type": "local", + }, } wf_format_tests = { "local minimum_format": { diff --git a/tests/integration/test_specs/default_worker_test.yaml b/tests/integration/test_specs/default_worker_test.yaml new file mode 100644 index 000000000..974921593 --- /dev/null +++ b/tests/integration/test_specs/default_worker_test.yaml @@ -0,0 +1,56 @@ +description: + name: multiple_workers + description: a very simple merlin workflow with multiple workers + +global.parameters: + GREET: + values : ["hello","hola"] + label : GREET.%% + WORLD: + values : ["world","mundo"] + label : WORLD.%% + +study: + - name: step_1 + description: say hello + run: + cmd: | + echo "$(GREET), $(WORLD)!" + task_queue: hello_queue + + - name: step_2 + description: step 2 + run: + cmd: | + echo "step_2" + depends: [step_1_*] + task_queue: echo_queue + + - name: step_3 + description: stop workers + run: + cmd: | + echo "stop workers" + depends: [step_2] + task_queue: other_queue + + - name: step_4 + description: another step + run: + cmd: | + echo "another step" + depends: [step_3] + task_queue: step_4_queue + +merlin: + resources: + workers: + step_1_merlin_test_worker: + args: -l INFO + steps: [step_1] + step_2_merlin_test_worker: + args: -l INFO + steps: [step_2] + other_merlin_test_worker: + args: -l INFO + steps: [step_3] diff --git a/tests/integration/test_specs/no_default_worker_test.yaml b/tests/integration/test_specs/no_default_worker_test.yaml new file mode 100644 index 000000000..b2f15d2ee --- /dev/null +++ b/tests/integration/test_specs/no_default_worker_test.yaml @@ -0,0 +1,56 @@ +description: + name: multiple_workers + description: a very simple merlin workflow with multiple workers + +global.parameters: + GREET: + values : ["hello","hola"] + label : GREET.%% + WORLD: + values : ["world","mundo"] + label : WORLD.%% + +study: + - name: step_1 + description: say hello + run: + cmd: | + echo "$(GREET), $(WORLD)!" + task_queue: hello_queue + + - name: step_2 + description: step 2 + run: + cmd: | + echo "step_2" + depends: [step_1_*] + task_queue: echo_queue + + - name: step_3 + description: stop workers + run: + cmd: | + echo "stop workers" + depends: [step_2] + task_queue: other_queue + + - name: step_4 + description: another step + run: + cmd: | + echo "another step" + depends: [step_3] + task_queue: step_4_queue + +merlin: + resources: + workers: + step_1_merlin_test_worker: + args: -l INFO + steps: [step_1] + step_2_merlin_test_worker: + args: -l INFO + steps: [step_2] + other_merlin_test_worker: + args: -l INFO + steps: [all] From f7508dd9dcf51dc2ada4ba8f64835466975f360f Mon Sep 17 00:00:00 2001 From: Brian Gunnarson Date: Thu, 4 May 2023 14:08:56 -0700 Subject: [PATCH 02/29] version bump and requirements fix --- CHANGELOG.md | 5 ++++- Makefile | 2 +- merlin/__init__.py | 4 ++-- merlin/ascii_art.py | 2 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 2 +- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- merlin/common/security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 2 +- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/celeryconfig.py | 2 +- merlin/config/configfile.py | 2 +- merlin/config/results_backend.py | 2 +- merlin/config/utils.py | 2 +- merlin/data/celery/__init__.py | 2 +- merlin/display.py | 2 +- merlin/examples/__init__.py | 2 +- merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- merlin/examples/workflows/feature_demo/requirements.txt | 2 +- .../examples/workflows/remote_feature_demo/requirements.txt | 2 +- merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 2 +- merlin/merlin_templates.py | 2 +- merlin/router.py | 2 +- merlin/server/__init__.py | 2 +- merlin/server/server_commands.py | 2 +- merlin/server/server_config.py | 2 +- merlin/server/server_util.py | 2 +- merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 2 +- merlin/spec/expansion.py | 2 +- merlin/spec/override.py | 2 +- merlin/spec/specification.py | 2 +- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 2 +- merlin/study/celeryadapter.py | 2 +- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 2 +- merlin/study/study.py | 2 +- merlin/utils.py | 2 +- setup.py | 2 +- tests/integration/conditions.py | 2 +- tests/integration/run_tests.py | 2 +- tests/integration/test_definitions.py | 2 +- 57 files changed, 61 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5ba817ef..15c9e0ffa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,13 +4,16 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [1.10.1] ### Fixed - A bug where assigning a worker all steps also assigned steps to the default worker ### Added - Tests to make sure the default worker is being assigned properly +### Changed +- Requirement name in examples/workflows/remote_feature_demo/requirements.txt and examples/workflows/feature_demo/requirements.txt from sklearn to scikit-learn since sklearn is now deprecated + ## [1.10.0] ### Fixed - Pip wheel wasn't including .sh files for merlin examples diff --git a/Makefile b/Makefile index 25266ace9..0153f10d4 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/__init__.py b/merlin/__init__.py index b4df0fa52..32bf5c875 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.10.0" +__version__ = "1.10.1" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index 02339d429..bb804d876 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index d09262adc..072c83b58 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index d90050599..1e9c75683 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index d90050599..1e9c75683 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index f35a75ae3..81c8865e5 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index dd12cfa93..f51055ab5 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index ceadfa5d2..65d503564 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index 34ccb07b0..635dd617b 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index 9f05e43d9..eb24e067e 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index d90050599..1e9c75683 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index 819fbc8e7..22c505df4 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index 13304a443..4d7dc176f 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index 15c915bf3..3b41cb459 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index 6d5f57cff..43dbd133a 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index 5ee7ed06e..aee1b62b0 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index 466bc56ee..0cf9b7c6d 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/celeryconfig.py b/merlin/config/celeryconfig.py index c2cc4aab5..2740aa0a1 100644 --- a/merlin/config/celeryconfig.py +++ b/merlin/config/celeryconfig.py @@ -10,7 +10,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index 38043c115..6aa38f9b2 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index de092b7a4..691bf5a6c 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/utils.py b/merlin/config/utils.py index f4cc093cc..9778cde3c 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index d90050599..1e9c75683 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index 0263e6c7c..89d14781e 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index d90050599..1e9c75683 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index 7181f055c..a66abf64a 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index 84c38d0b2..b859ff605 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/workflows/feature_demo/requirements.txt b/merlin/examples/workflows/feature_demo/requirements.txt index 93909e8aa..e308e1895 100644 --- a/merlin/examples/workflows/feature_demo/requirements.txt +++ b/merlin/examples/workflows/feature_demo/requirements.txt @@ -1,2 +1,2 @@ -sklearn +scikit-learn merlin-spellbook diff --git a/merlin/examples/workflows/remote_feature_demo/requirements.txt b/merlin/examples/workflows/remote_feature_demo/requirements.txt index 93909e8aa..e308e1895 100644 --- a/merlin/examples/workflows/remote_feature_demo/requirements.txt +++ b/merlin/examples/workflows/remote_feature_demo/requirements.txt @@ -1,2 +1,2 @@ -sklearn +scikit-learn merlin-spellbook diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index 5273fb8e7..cfbc21e38 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index 8dc3a83e9..580952b1d 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index 41dbb05e3..f49d3cc94 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index 01a86b2ae..4809c7aee 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/router.py b/merlin/router.py index a48f86ed4..160909ddf 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/__init__.py b/merlin/server/__init__.py index 6fae626c9..7c6148fee 100644 --- a/merlin/server/__init__.py +++ b/merlin/server/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index e0a481171..d53463c2b 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index 445495802..14abfb0d4 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index 7a5da16f5..d101af2f0 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index d90050599..1e9c75683 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index 858c8401e..f58731336 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index e2dc3f651..b20ce7d09 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index a535c7bd9..38f03d6e9 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/override.py b/merlin/spec/override.py index 7862993fe..0f1e71f62 100644 --- a/merlin/spec/override.py +++ b/merlin/spec/override.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 9f3a6c16f..8d36297ab 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index d90050599..1e9c75683 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index e05342fd1..eeaead5ee 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 7c2e42b72..4c01cfd73 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index bdc508f87..a85a86e47 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index e637c0939..2df053051 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index 7302344ff..14f273dfa 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index 9ee0c131d..3a51c926e 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/utils.py b/merlin/utils.py index ed7f12b9a..7c18407e7 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/setup.py b/setup.py index efbc5a529..9693b06ba 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index 22d911e7e..db21e5429 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index c858649c8..9b0270d3b 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index 2b990ea40..093644f9f 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.0. +# This file is part of Merlin, Version: 1.10.1. # # For details, see https://github.com/LLNL/merlin. # From 98e06f6557774f9cac005d03511151f567c17f56 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Thu, 25 May 2023 13:05:39 -0700 Subject: [PATCH 03/29] Bugfix/filename-special-vars (#425) * fix file naming bug * fix filename bug with variable as study name * add tests for the file name special vars changes * modify changelog * implement Luc's suggestions * remove replace line --- CHANGELOG.md | 11 ++++++ merlin/study/study.py | 57 +++++++++++++-------------- tests/integration/conditions.py | 12 ++++-- tests/integration/test_definitions.py | 24 +++++++---- tests/unit/study/test_study.py | 50 ++++++++++++++++++++++- 5 files changed, 111 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15c9e0ffa..603086aaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [unreleased] +### Fixed +- A bug where the .orig, .partial, and .expanded file names were using the study name rather than the original file name + +### Added +- Tests for ensuring `$(MERLIN_SPEC_ORIGINAL_TEMPLATE)`, `$(MERLIN_SPEC_ARCHIVED_COPY)`, and `$(MERLIN_SPEC_EXECUTED_RUN)` are stored correctly + +### Changed +- The ProvenanceYAMLFileHasRegex condition for integration tests now saves the study name and spec file name as attributes instead of just the study name + - This lead to minor changes in 3 tests ("local override feature demo", "local pgen feature demo", and "remote feature demo") with what we pass to this specific condition + ## [1.10.1] ### Fixed - A bug where assigning a worker all steps also assigned steps to the default worker diff --git a/merlin/study/study.py b/merlin/study/study.py index 3a51c926e..5446c84a1 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -36,6 +36,7 @@ import time from contextlib import suppress from copy import deepcopy +from pathlib import Path from cached_property import cached_property from maestrowf.datastructures.core import Study @@ -83,6 +84,7 @@ def __init__( # pylint: disable=R0913 pgen_file=None, pargs=None, ): + self.filepath = filepath self.original_spec = MerlinSpec.load_specification(filepath) self.override_vars = override_vars error_override_vars(self.override_vars, self.original_spec.path) @@ -114,19 +116,8 @@ def __init__( # pylint: disable=R0913 # below will be substituted for sample values on execution "MERLIN_SAMPLE_VECTOR": " ".join([f"$({k})" for k in self.get_sample_labels(from_spec=self.original_spec)]), "MERLIN_SAMPLE_NAMES": " ".join(self.get_sample_labels(from_spec=self.original_spec)), - "MERLIN_SPEC_ORIGINAL_TEMPLATE": os.path.join( - self.info, - self.original_spec.description["name"].replace(" ", "_") + ".orig.yaml", - ), - "MERLIN_SPEC_EXECUTED_RUN": os.path.join( - self.info, - self.original_spec.description["name"].replace(" ", "_") + ".partial.yaml", - ), - "MERLIN_SPEC_ARCHIVED_COPY": os.path.join( - self.info, - self.original_spec.description["name"].replace(" ", "_") + ".expanded.yaml", - ), } + self._set_special_file_vars() self.pgen_file = pgen_file self.pargs = pargs @@ -134,12 +125,27 @@ def __init__( # pylint: disable=R0913 self.dag = None self.load_dag() - def write_original_spec(self, filename): + def _set_special_file_vars(self): + """Setter for the orig, partial, and expanded file paths of a study.""" + base_name = Path(self.filepath).stem + self.special_vars["MERLIN_SPEC_ORIGINAL_TEMPLATE"] = os.path.join( + self.info, + base_name + ".orig.yaml", + ) + self.special_vars["MERLIN_SPEC_EXECUTED_RUN"] = os.path.join( + self.info, + base_name + ".partial.yaml", + ) + self.special_vars["MERLIN_SPEC_ARCHIVED_COPY"] = os.path.join( + self.info, + base_name + ".expanded.yaml", + ) + + def write_original_spec(self): """ - Copy the original spec into merlin_info/ as '.orig.yaml'. + Copy the original spec into merlin_info/ as '.orig.yaml'. """ - spec_name = os.path.join(self.info, filename + ".orig.yaml") - shutil.copyfile(self.original_spec.path, spec_name) + shutil.copyfile(self.original_spec.path, self.special_vars["MERLIN_SPEC_ORIGINAL_TEMPLATE"]) def label_clash_error(self): """ @@ -368,10 +374,6 @@ def expanded_spec(self): return self.get_expanded_spec() result = self.get_expanded_spec() - expanded_name = result.description["name"].replace(" ", "_") + ".expanded.yaml" - - # Set expanded filepath - expanded_filepath = os.path.join(self.info, expanded_name) # expand provenance spec filename if contains_token(self.original_spec.name) or contains_shell_ref(self.original_spec.name): @@ -394,8 +396,8 @@ def expanded_spec(self): self.workspace = expanded_workspace self.info = os.path.join(self.workspace, "merlin_info") self.special_vars["MERLIN_INFO"] = self.info + self._set_special_file_vars() - expanded_filepath = os.path.join(self.info, expanded_name) new_spec_text = expand_by_line(result.dump(), MerlinStudy.get_user_vars(result)) result = MerlinSpec.load_spec_from_string(new_spec_text) result = expand_env_vars(result) @@ -412,15 +414,13 @@ def expanded_spec(self): os.path.join(self.info, os.path.basename(self.samples_file)), ) - # write expanded spec for provenance - with open(expanded_filepath, "w") as f: # pylint: disable=C0103 + # write expanded spec for provenance and set the path (necessary for testing) + with open(self.special_vars["MERLIN_SPEC_ARCHIVED_COPY"], "w") as f: # pylint: disable=C0103 f.write(result.dump()) + result.path = self.special_vars["MERLIN_SPEC_ARCHIVED_COPY"] # write original spec for provenance - result = MerlinSpec.load_spec_from_string(result.dump()) - result.path = expanded_filepath - name = result.description["name"].replace(" ", "_") - self.write_original_spec(name) + self.write_original_spec() # write partially-expanded spec for provenance partial_spec = deepcopy(self.original_spec) @@ -428,8 +428,7 @@ def expanded_spec(self): partial_spec.environment["variables"] = result.environment["variables"] if "labels" in result.environment: partial_spec.environment["labels"] = result.environment["labels"] - partial_spec_path = os.path.join(self.info, name + ".partial.yaml") - with open(partial_spec_path, "w") as f: # pylint: disable=C0103 + with open(self.special_vars["MERLIN_SPEC_EXECUTED_RUN"], "w") as f: # pylint: disable=C0103 f.write(partial_spec.dump()) LOG.info(f"Study workspace is '{self.workspace}'.") diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index db21e5429..4da8c36a1 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -249,14 +249,16 @@ class ProvenanceYAMLFileHasRegex(HasRegex): MUST contain a given regular expression. """ - def __init__(self, regex, name, output_path, provenance_type, negate=False): # pylint: disable=R0913 + def __init__(self, regex, spec_file_name, study_name, output_path, provenance_type, negate=False): # pylint: disable=R0913 """ :param `regex`: a string regex pattern - :param `name`: the name of a study + :param `spec_file_name`: the name of the spec file + :param `study_name`: the name of a study :param `output_path`: the $(OUTPUT_PATH) of a study """ super().__init__(regex, negate=negate) - self.name = name + self.spec_file_name = spec_file_name + self.study_name = study_name self.output_path = output_path provenance_types = ["orig", "partial", "expanded"] if provenance_type not in provenance_types: @@ -277,7 +279,9 @@ def glob_string(self): """ Returns a regex string for the glob library to recursively find files with. """ - return f"{self.output_path}/{self.name}" f"_[0-9]*-[0-9]*/merlin_info/{self.name}.{self.prov_type}.yaml" + return ( + f"{self.output_path}/{self.study_name}" f"_[0-9]*-[0-9]*/merlin_info/{self.spec_file_name}.{self.prov_type}.yaml" + ) def is_within(self): # pylint: disable=W0221 """ diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index 093644f9f..cf7c008de 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -435,31 +435,36 @@ def define_tests(): # pylint: disable=R0914,R0915 HasReturnCode(), ProvenanceYAMLFileHasRegex( regex=r"HELLO: \$\(SCRIPTS\)/hello_world.py", - name="feature_demo", + spec_file_name="feature_demo", + study_name="feature_demo", output_path=OUTPUT_DIR, provenance_type="orig", ), ProvenanceYAMLFileHasRegex( regex=r"name: \$\(NAME\)", - name="feature_demo", + spec_file_name="feature_demo", + study_name="feature_demo", output_path=OUTPUT_DIR, provenance_type="partial", ), ProvenanceYAMLFileHasRegex( regex="studies/feature_demo_", - name="feature_demo", + spec_file_name="feature_demo", + study_name="feature_demo", output_path=OUTPUT_DIR, provenance_type="partial", ), ProvenanceYAMLFileHasRegex( regex="name: feature_demo", - name="feature_demo", + spec_file_name="feature_demo", + study_name="feature_demo", output_path=OUTPUT_DIR, provenance_type="expanded", ), ProvenanceYAMLFileHasRegex( regex=r"\$\(NAME\)", - name="feature_demo", + spec_file_name="feature_demo", + study_name="feature_demo", output_path=OUTPUT_DIR, provenance_type="expanded", negate=True, @@ -510,13 +515,15 @@ def define_tests(): # pylint: disable=R0914,R0915 "conditions": [ ProvenanceYAMLFileHasRegex( regex=r"\[0.3333333", - name="feature_demo", + spec_file_name="feature_demo", + study_name="feature_demo", output_path=OUTPUT_DIR, provenance_type="expanded", ), ProvenanceYAMLFileHasRegex( regex=r"\[0.5", - name="feature_demo", + spec_file_name="feature_demo", + study_name="feature_demo", output_path=OUTPUT_DIR, provenance_type="expanded", negate=True, @@ -715,7 +722,8 @@ def define_tests(): # pylint: disable=R0914,R0915 HasReturnCode(), ProvenanceYAMLFileHasRegex( regex="cli_test_demo_workers:", - name="feature_demo", + spec_file_name="remote_feature_demo", + study_name="feature_demo", output_path=OUTPUT_DIR, provenance_type="expanded", ), diff --git a/tests/unit/study/test_study.py b/tests/unit/study/test_study.py index a00995d55..cb15805cd 100644 --- a/tests/unit/study/test_study.py +++ b/tests/unit/study/test_study.py @@ -41,6 +41,15 @@ nodes: 1 task_queue: hello_queue + - name: test_special_vars + description: test the special vars + run: + cmd: | + echo $(MERLIN_SPEC_ORIGINAL_TEMPLATE) + echo $(MERLIN_SPEC_EXECUTED_RUN) + echo $(MERLIN_SPEC_ARCHIVED_COPY) + task_queue: special_var_queue + global.parameters: X2: values : [0.5] @@ -234,11 +243,16 @@ class TestMerlinStudy(unittest.TestCase): @staticmethod def file_contains_string(f, string): - return string in open(f, "r").read() + result = False + with open(f, "r") as infile: + if string in infile.read(): + result = True + return result def setUp(self): self.tmpdir = tempfile.mkdtemp() - self.merlin_spec_filepath = os.path.join(self.tmpdir, "basic_ensemble.yaml") + self.base_name = "basic_ensemble" + self.merlin_spec_filepath = os.path.join(self.tmpdir, f"{self.base_name}.yaml") with open(self.merlin_spec_filepath, "w+") as _file: _file.write(MERLIN_SPEC) @@ -263,6 +277,34 @@ def test_expanded_spec(self): assert TestMerlinStudy.file_contains_string(self.study.expanded_spec.path, "$PATH") assert not TestMerlinStudy.file_contains_string(self.study.expanded_spec.path, "PATH_VAR: $PATH") + # Special vars are in the second step of MERLIN_SPEC so grab that step here + original_special_var_step = self.study.original_spec.study[1]["run"]["cmd"] + expanded_special_var_step = self.study.expanded_spec.study[1]["run"]["cmd"] + + # Make sure the special filepath variables aren't expanded in the original spec + assert "$(MERLIN_SPEC_ORIGINAL_TEMPLATE)" in original_special_var_step + assert "$(MERLIN_SPEC_EXECUTED_RUN)" in original_special_var_step + assert "$(MERLIN_SPEC_ARCHIVED_COPY)" in original_special_var_step + + # Make sure the special filepath variables aren't left in their variable form in the expanded spec + assert "$(MERLIN_SPEC_ORIGINAL_TEMPLATE)" not in expanded_special_var_step + assert "$(MERLIN_SPEC_EXECUTED_RUN)" not in expanded_special_var_step + assert "$(MERLIN_SPEC_ARCHIVED_COPY)" not in expanded_special_var_step + + # Make sure the special filepath variables we're expanded appropriately in the expanded spec + assert ( + f"{self.base_name}.orig.yaml" in expanded_special_var_step + and "unit_test1.orig.yaml" not in expanded_special_var_step + ) + assert ( + f"{self.base_name}.partial.yaml" in expanded_special_var_step + and "unit_test1.partial.yaml" not in expanded_special_var_step + ) + assert ( + f"{self.base_name}.expanded.yaml" in expanded_special_var_step + and "unit_test1.expanded.yaml" not in expanded_special_var_step + ) + def test_column_label_conflict(self): """ If there is a common key between Maestro's global.parameters and @@ -291,3 +333,7 @@ def test_no_env(self): assert isinstance(study_no_env, MerlinStudy), bad_type_err except Exception as e: assert False, f"Encountered unexpected exception, {e}, for viable MerlinSpec without optional 'env' section." + + +if __name__ == "__main__": + unittest.main() From b8dd2b2bc69657601b49e07a026f9e1d3c74fc71 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Fri, 26 May 2023 08:25:54 -0700 Subject: [PATCH 04/29] Create dependabot-changelog-updater.yml --- .../dependabot-changelog-updater.yml | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/dependabot-changelog-updater.yml diff --git a/.github/workflows/dependabot-changelog-updater.yml b/.github/workflows/dependabot-changelog-updater.yml new file mode 100644 index 000000000..1677cf050 --- /dev/null +++ b/.github/workflows/dependabot-changelog-updater.yml @@ -0,0 +1,34 @@ +# See https://github.com/dangoslen/dependabot-changelog-helper for more info +name: Dependabot Changelog Updater +on: + pull_request: + types: + - opened + - synchronize + - reopened + - ready_for_review + - labeled + - unlabeled + +jobs: + changelog: + runs-on: ubuntu-latest + if: ${{ github.actor == 'dependabot[bot]' }} + steps: + - uses: actions/checkout@v3 + with: + # Depending on your needs, you can use a token that will re-trigger workflows + # See https://github.com/stefanzweifel/git-auto-commit-action#commits-of-this-action-do-not-trigger-new-workflow-runs + token: ${{ secrets.GITHUB_TOKEN }} + + - uses: dangoslen/dependabot-changelog-helper@v3 + with: + version: ${{ needs.setup.outputs.version }} + activationLabel: 'dependabot' + changelogPath: './CHANGELOG.md' + + # This step is required for committing the changes to your branch. + # See https://github.com/stefanzweifel/git-auto-commit-action#commits-of-this-action-do-not-trigger-new-workflow-runs + - uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: "Updated Changelog" From 4c404235c6c5cf775a9db9606413c4a9cfb0fb52 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Fri, 26 May 2023 08:40:45 -0700 Subject: [PATCH 05/29] testing outputs of modifying changelog --- .../dependabot-changelog-updater.yml | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/dependabot-changelog-updater.yml b/.github/workflows/dependabot-changelog-updater.yml index 1677cf050..a8afccc92 100644 --- a/.github/workflows/dependabot-changelog-updater.yml +++ b/.github/workflows/dependabot-changelog-updater.yml @@ -11,24 +11,30 @@ on: - unlabeled jobs: - changelog: + changelog-updater: runs-on: ubuntu-latest if: ${{ github.actor == 'dependabot[bot]' }} steps: - - uses: actions/checkout@v3 - with: - # Depending on your needs, you can use a token that will re-trigger workflows - # See https://github.com/stefanzweifel/git-auto-commit-action#commits-of-this-action-do-not-trigger-new-workflow-runs - token: ${{ secrets.GITHUB_TOKEN }} +# - uses: actions/checkout@v3 +# with: +# # Depending on your needs, you can use a token that will re-trigger workflows +# # See https://github.com/stefanzweifel/git-auto-commit-action#commits-of-this-action-do-not-trigger-new-workflow-runs +# token: ${{ secrets.GITHUB_TOKEN }} - - uses: dangoslen/dependabot-changelog-helper@v3 + - name: modify changelog + id: modify-changelog + uses: dangoslen/dependabot-changelog-helper@v3 with: version: ${{ needs.setup.outputs.version }} activationLabel: 'dependabot' changelogPath: './CHANGELOG.md' + + - name: commit changes + run: | + echo ${{ steps.modify-changelog.outputs }} # This step is required for committing the changes to your branch. # See https://github.com/stefanzweifel/git-auto-commit-action#commits-of-this-action-do-not-trigger-new-workflow-runs - - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: "Updated Changelog" +# - uses: stefanzweifel/git-auto-commit-action@v4 +# with: +# commit_message: "Updated Changelog" From 2d231fb937303839934d4ee2048d554252ce9dbd Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Fri, 26 May 2023 09:13:57 -0700 Subject: [PATCH 06/29] delete dependabot-changelog-updater --- .../dependabot-changelog-updater.yml | 40 ------------------- 1 file changed, 40 deletions(-) delete mode 100644 .github/workflows/dependabot-changelog-updater.yml diff --git a/.github/workflows/dependabot-changelog-updater.yml b/.github/workflows/dependabot-changelog-updater.yml deleted file mode 100644 index a8afccc92..000000000 --- a/.github/workflows/dependabot-changelog-updater.yml +++ /dev/null @@ -1,40 +0,0 @@ -# See https://github.com/dangoslen/dependabot-changelog-helper for more info -name: Dependabot Changelog Updater -on: - pull_request: - types: - - opened - - synchronize - - reopened - - ready_for_review - - labeled - - unlabeled - -jobs: - changelog-updater: - runs-on: ubuntu-latest - if: ${{ github.actor == 'dependabot[bot]' }} - steps: -# - uses: actions/checkout@v3 -# with: -# # Depending on your needs, you can use a token that will re-trigger workflows -# # See https://github.com/stefanzweifel/git-auto-commit-action#commits-of-this-action-do-not-trigger-new-workflow-runs -# token: ${{ secrets.GITHUB_TOKEN }} - - - name: modify changelog - id: modify-changelog - uses: dangoslen/dependabot-changelog-helper@v3 - with: - version: ${{ needs.setup.outputs.version }} - activationLabel: 'dependabot' - changelogPath: './CHANGELOG.md' - - - name: commit changes - run: | - echo ${{ steps.modify-changelog.outputs }} - - # This step is required for committing the changes to your branch. - # See https://github.com/stefanzweifel/git-auto-commit-action#commits-of-this-action-do-not-trigger-new-workflow-runs -# - uses: stefanzweifel/git-auto-commit-action@v4 -# with: -# commit_message: "Updated Changelog" From 6b142d9b0f67f2cca6856b722b96114ef7a2ca7b Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Wed, 28 Jun 2023 07:56:45 -0700 Subject: [PATCH 07/29] feature/pdf-docs (#427) * first attempt at adding pdf * fixing build error * modify changelog to show docs changes * fix errors Luc found in the build logs * trying out removal of latex * reverting latex changes back * uncommenting the latex_elements settings * adding epub to see if latex will build * adding a latex engine variable to conf * fix naming error with latex_engine * attempting to add a logo to the pdf build * testing an override to the searchtools file * revert back to not using searchtools override * update changelog --- .readthedocs.yaml | 2 ++ CHANGELOG.md | 3 +++ docs/source/conf.py | 13 ++++++++----- docs/source/faq.rst | 2 +- docs/source/merlin_developer.rst | 3 +-- docs/source/merlin_variables.rst | 2 +- docs/source/modules/installation/installation.rst | 2 +- 7 files changed, 17 insertions(+), 10 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index c1c252e30..ee62bcb5e 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -11,3 +11,5 @@ sphinx: python: install: - requirements: docs/requirements.txt + +formats: [pdf] \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 603086aaa..25ee73963 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,13 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [unreleased] ### Fixed - A bug where the .orig, .partial, and .expanded file names were using the study name rather than the original file name +- Some build warnings in the docs (unknown targets, duplicate targets, title underlines too short, etc.) ### Added - Tests for ensuring `$(MERLIN_SPEC_ORIGINAL_TEMPLATE)`, `$(MERLIN_SPEC_ARCHIVED_COPY)`, and `$(MERLIN_SPEC_EXECUTED_RUN)` are stored correctly +- A pdf download format for the docs ### Changed - The ProvenanceYAMLFileHasRegex condition for integration tests now saves the study name and spec file name as attributes instead of just the study name - This lead to minor changes in 3 tests ("local override feature demo", "local pgen feature demo", and "remote feature demo") with what we pass to this specific condition +- Uncommented Latex support in the docs configuration to get pdf builds working ## [1.10.1] ### Fixed diff --git a/docs/source/conf.py b/docs/source/conf.py index 315978a6a..b578e8672 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -66,7 +66,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -122,21 +122,24 @@ # -- Options for LaTeX output ------------------------------------------------ +latex_engine = "pdflatex" latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # - # 'papersize': 'letterpaper', + 'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). # - # 'pointsize': '10pt', + 'pointsize': '10pt', # Additional stuff for the LaTeX preamble. # - # 'preamble': '', + 'preamble': '', # Latex figure (float) alignment # - # 'figure_align': 'htbp', + 'figure_align': 'htbp', } +latex_logo = "../images/merlin.png" + # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). diff --git a/docs/source/faq.rst b/docs/source/faq.rst index d0ef8e109..3632aab6c 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -383,7 +383,7 @@ in the ``launch_args`` variable in the batch section. What is PBS? ~~~~~~~~~~~~ Another job scheduler. See `Portable Batch System -https://en.wikipedia.org/wiki/Portable_Batch_System`_ +`_ . This functionality is only available to launch a flux scheduler. diff --git a/docs/source/merlin_developer.rst b/docs/source/merlin_developer.rst index 08947ca89..e88352f3e 100644 --- a/docs/source/merlin_developer.rst +++ b/docs/source/merlin_developer.rst @@ -164,8 +164,7 @@ properties labeled ``name`` and ``population`` that are both required, it would } Here, ``name`` can only be a string but ``population`` can be both a string and an integer. -For help with json schema formatting, check out the `step-by-step getting started guide -`_. +For help with json schema formatting, check out the `step-by-step getting started guide`_. The next step is to enable this block in the schema validation process. To do this we need to: diff --git a/docs/source/merlin_variables.rst b/docs/source/merlin_variables.rst index 7f545a4d2..d67e06412 100644 --- a/docs/source/merlin_variables.rst +++ b/docs/source/merlin_variables.rst @@ -161,7 +161,7 @@ Reserved variables $(MERLIN_INFO)/*.expanded.yaml The ``LAUNCHER`` Variable -+++++++++++++++++++++ ++++++++++++++++++++++++++ ``$(LAUNCHER)`` is a special case of a reserved variable since it's value *can* be changed. It serves as an abstraction to launch a job with parallel schedulers like :ref:`slurm`, diff --git a/docs/source/modules/installation/installation.rst b/docs/source/modules/installation/installation.rst index d18261af5..96195ff3d 100644 --- a/docs/source/modules/installation/installation.rst +++ b/docs/source/modules/installation/installation.rst @@ -229,7 +229,7 @@ If everything is set up correctly, you should see: (OPTIONAL) Docker Advanced Installation ----------------------------- +--------------------------------------- RabbitMQ Server +++++++++++++++ From e3e1a307d6c65fcc834fe4a3aba2271f9109932d Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Wed, 28 Jun 2023 10:18:25 -0700 Subject: [PATCH 08/29] bugfix/openfoam_singularity_issues (#426) * fix openfoam_singularity issues * update requirements and descriptions for openfoam examples --- CHANGELOG.md | 2 ++ merlin/examples/generator.py | 2 ++ merlin/examples/workflows/openfoam_wf/openfoam_wf.yaml | 1 + merlin/examples/workflows/openfoam_wf/openfoam_wf_template.yaml | 1 + merlin/examples/workflows/openfoam_wf/requirements.txt | 2 +- .../workflows/openfoam_wf_no_docker/openfoam_wf_no_docker.yaml | 1 + .../openfoam_wf_no_docker/openfoam_wf_no_docker_template.yaml | 1 + .../examples/workflows/openfoam_wf_no_docker/requirements.txt | 2 +- .../{openfoam_wf.yaml => openfoam_wf_singularity.yaml} | 1 + .../examples/workflows/openfoam_wf_singularity/requirements.txt | 2 +- 10 files changed, 12 insertions(+), 3 deletions(-) rename merlin/examples/workflows/openfoam_wf_singularity/{openfoam_wf.yaml => openfoam_wf_singularity.yaml} (99%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25ee73963..ebaea86f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [unreleased] ### Fixed - A bug where the .orig, .partial, and .expanded file names were using the study name rather than the original file name +- A bug where the openfoam_wf_singularity example was not being found - Some build warnings in the docs (unknown targets, duplicate targets, title underlines too short, etc.) ### Added @@ -16,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - The ProvenanceYAMLFileHasRegex condition for integration tests now saves the study name and spec file name as attributes instead of just the study name - This lead to minor changes in 3 tests ("local override feature demo", "local pgen feature demo", and "remote feature demo") with what we pass to this specific condition +- Updated scikit-learn requirement for the openfoam_wf_singularity example - Uncommented Latex support in the docs configuration to get pdf builds working ## [1.10.1] diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index b859ff605..308785784 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -83,6 +83,8 @@ def list_examples(): directory = os.path.join(os.path.join(EXAMPLES_DIR, example_dir), "") specs = glob.glob(directory + "*.yaml") for spec in specs: + if "template" in spec: + continue with open(spec) as f: # pylint: disable=C0103 try: spec_metadata = yaml.safe_load(f)["description"] diff --git a/merlin/examples/workflows/openfoam_wf/openfoam_wf.yaml b/merlin/examples/workflows/openfoam_wf/openfoam_wf.yaml index e6dabc8c2..0d98445c0 100644 --- a/merlin/examples/workflows/openfoam_wf/openfoam_wf.yaml +++ b/merlin/examples/workflows/openfoam_wf/openfoam_wf.yaml @@ -3,6 +3,7 @@ description: description: | A parameter study that includes initializing, running, post-processing, collecting, learning and visualizing OpenFOAM runs + using docker. env: diff --git a/merlin/examples/workflows/openfoam_wf/openfoam_wf_template.yaml b/merlin/examples/workflows/openfoam_wf/openfoam_wf_template.yaml index 96e13064b..64433d3af 100644 --- a/merlin/examples/workflows/openfoam_wf/openfoam_wf_template.yaml +++ b/merlin/examples/workflows/openfoam_wf/openfoam_wf_template.yaml @@ -3,6 +3,7 @@ description: description: | A parameter study that includes initializing, running, post-processing, collecting, learning and visualizing OpenFOAM runs + using docker. env: diff --git a/merlin/examples/workflows/openfoam_wf/requirements.txt b/merlin/examples/workflows/openfoam_wf/requirements.txt index 8042c2422..ef63ca016 100644 --- a/merlin/examples/workflows/openfoam_wf/requirements.txt +++ b/merlin/examples/workflows/openfoam_wf/requirements.txt @@ -1,3 +1,3 @@ Ofpp==0.11 -scikit-learn==0.21.3 +scikit-learn>=1.0.2 matplotlib==3.1.1 diff --git a/merlin/examples/workflows/openfoam_wf_no_docker/openfoam_wf_no_docker.yaml b/merlin/examples/workflows/openfoam_wf_no_docker/openfoam_wf_no_docker.yaml index ab8224029..688e8ceff 100644 --- a/merlin/examples/workflows/openfoam_wf_no_docker/openfoam_wf_no_docker.yaml +++ b/merlin/examples/workflows/openfoam_wf_no_docker/openfoam_wf_no_docker.yaml @@ -3,6 +3,7 @@ description: description: | A parameter study that includes initializing, running, post-processing, collecting, learning and vizualizing OpenFOAM runs + without using docker. env: diff --git a/merlin/examples/workflows/openfoam_wf_no_docker/openfoam_wf_no_docker_template.yaml b/merlin/examples/workflows/openfoam_wf_no_docker/openfoam_wf_no_docker_template.yaml index 084a1c0bb..3fcbc3588 100644 --- a/merlin/examples/workflows/openfoam_wf_no_docker/openfoam_wf_no_docker_template.yaml +++ b/merlin/examples/workflows/openfoam_wf_no_docker/openfoam_wf_no_docker_template.yaml @@ -3,6 +3,7 @@ description: description: | A parameter study that includes initializing, running, post-processing, collecting, learning and vizualizing OpenFOAM runs + without using docker. env: diff --git a/merlin/examples/workflows/openfoam_wf_no_docker/requirements.txt b/merlin/examples/workflows/openfoam_wf_no_docker/requirements.txt index 8042c2422..ef63ca016 100644 --- a/merlin/examples/workflows/openfoam_wf_no_docker/requirements.txt +++ b/merlin/examples/workflows/openfoam_wf_no_docker/requirements.txt @@ -1,3 +1,3 @@ Ofpp==0.11 -scikit-learn==0.21.3 +scikit-learn>=1.0.2 matplotlib==3.1.1 diff --git a/merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf.yaml b/merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf_singularity.yaml similarity index 99% rename from merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf.yaml rename to merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf_singularity.yaml index 3f3bbb672..03837bcb3 100644 --- a/merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf.yaml +++ b/merlin/examples/workflows/openfoam_wf_singularity/openfoam_wf_singularity.yaml @@ -3,6 +3,7 @@ description: description: | A parameter study that includes initializing, running, post-processing, collecting, learning and visualizing OpenFOAM runs + using singularity. env: diff --git a/merlin/examples/workflows/openfoam_wf_singularity/requirements.txt b/merlin/examples/workflows/openfoam_wf_singularity/requirements.txt index 8042c2422..ef63ca016 100644 --- a/merlin/examples/workflows/openfoam_wf_singularity/requirements.txt +++ b/merlin/examples/workflows/openfoam_wf_singularity/requirements.txt @@ -1,3 +1,3 @@ Ofpp==0.11 -scikit-learn==0.21.3 +scikit-learn>=1.0.2 matplotlib==3.1.1 From 12ff3d782d30579b1387eb314f33924d33d1877f Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Tue, 11 Jul 2023 11:26:15 -0700 Subject: [PATCH 09/29] bugfix/output-path-substitution (#430) * fix bug with output_path and variable substitution * add tests for cli substitutions --- CHANGELOG.md | 2 ++ merlin/study/study.py | 15 +++++++++++-- tests/integration/test_definitions.py | 22 +++++++++++++++++++ .../test_specs/cli_substitution_test.yaml | 14 ++++++++++++ 4 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 tests/integration/test_specs/cli_substitution_test.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index ebaea86f2..d1452104b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - A bug where the .orig, .partial, and .expanded file names were using the study name rather than the original file name - A bug where the openfoam_wf_singularity example was not being found - Some build warnings in the docs (unknown targets, duplicate targets, title underlines too short, etc.) +- A bug where when the output path contained a variable that was overridden, the overridden variable was not changed in the output_path ### Added - Tests for ensuring `$(MERLIN_SPEC_ORIGINAL_TEMPLATE)`, `$(MERLIN_SPEC_ARCHIVED_COPY)`, and `$(MERLIN_SPEC_EXECUTED_RUN)` are stored correctly - A pdf download format for the docs +- Tests for cli substitutions ### Changed - The ProvenanceYAMLFileHasRegex condition for integration tests now saves the study name and spec file name as attributes instead of just the study name diff --git a/merlin/study/study.py b/merlin/study/study.py index 5446c84a1..74b7c3181 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -306,8 +306,17 @@ def output_path(self): output_path = str(self.original_spec.output_path) - if (self.override_vars is not None) and ("OUTPUT_PATH" in self.override_vars): - output_path = str(self.override_vars["OUTPUT_PATH"]) + # If there are override vars we need to check that the output path doesn't need changed + if self.override_vars is not None: + # Case where output path is directly modified + if "OUTPUT_PATH" in self.override_vars: + output_path = str(self.override_vars["OUTPUT_PATH"]) + else: + for var_name, var_val in self.override_vars.items(): + token = f"$({var_name})" + # Case where output path contains a variable that was overridden + if token in output_path: + output_path = output_path.replace(token, str(var_val)) output_path = expand_line(output_path, self.user_vars, env_vars=True) output_path = os.path.abspath(output_path) @@ -315,6 +324,8 @@ def output_path(self): os.makedirs(output_path) LOG.info(f"Made dir(s) to output path '{output_path}'.") + LOG.info(f"OUTPUT_PATH: {os.path.basename(output_path)}") + return output_path @cached_property diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index cf7c008de..d20efb329 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -127,6 +127,7 @@ def define_tests(): # pylint: disable=R0914,R0915 flux_native = f"{test_specs}/flux_par_native_test.yaml" lsf = f"{examples}/lsf/lsf_par.yaml" mul_workers_demo = f"{dev_examples}/multiple_workers.yaml" + cli_substitution_wf = f"{test_specs}/cli_substitution_test.yaml" # Other shortcuts black = "black --check --target-version py36" @@ -323,6 +324,26 @@ def define_tests(): # pylint: disable=R0914,R0915 "run type": "local", }, } + cli_substitution_tests = { + "no substitutions": { + "cmds": f"merlin run {cli_substitution_wf} --local", + "conditions": [HasReturnCode(), HasRegex(r"OUTPUT_PATH: output_path_no_substitution")], + "run type": "local", + "cleanup": "rm -r output_path_no_substitution", + }, + "output_path substitution": { + "cmds": f"merlin run {cli_substitution_wf} --local --vars OUTPUT_PATH=output_path_substitution", + "conditions": [HasReturnCode(), HasRegex(r"OUTPUT_PATH: output_path_substitution")], + "run type": "local", + "cleanup": "rm -r output_path_substitution", + }, + "output_path w/ variable substitution": { + "cmds": f"merlin run {cli_substitution_wf} --local --vars SUB=variable_sub", + "conditions": [HasReturnCode(), HasRegex(r"OUTPUT_PATH: output_path_variable_sub")], + "run type": "local", + "cleanup": "rm -r output_path_variable_sub", + }, + } example_tests = { "example failure": {"cmds": "merlin example failure", "conditions": HasRegex("not found"), "run type": "local"}, "example simple_chain": { @@ -748,6 +769,7 @@ def define_tests(): # pylint: disable=R0914,R0915 examples_check, run_workers_echo_tests, wf_format_tests, + cli_substitution_tests, example_tests, restart_step_tests, restart_wf_tests, diff --git a/tests/integration/test_specs/cli_substitution_test.yaml b/tests/integration/test_specs/cli_substitution_test.yaml new file mode 100644 index 000000000..5cbbb70d3 --- /dev/null +++ b/tests/integration/test_specs/cli_substitution_test.yaml @@ -0,0 +1,14 @@ +description: + name: cli_substitution_test + description: a spec that helps test cli substitutions + +env: + variables: + SUB: no_substitution + OUTPUT_PATH: output_path_$(SUB) + +study: + - name: step1 + description: step 1 + run: + cmd: echo "test" From 5c69c0beaf74962abd585ea321a6f071d28ad881 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Fri, 4 Aug 2023 15:28:55 -0700 Subject: [PATCH 10/29] bugfix/scheduler-permission-error (#436) --- CHANGELOG.md | 1 + merlin/study/batch.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d1452104b..981ce7198 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - A bug where the openfoam_wf_singularity example was not being found - Some build warnings in the docs (unknown targets, duplicate targets, title underlines too short, etc.) - A bug where when the output path contained a variable that was overridden, the overridden variable was not changed in the output_path +- A bug where permission denied errors happened when checking for system scheduler ### Added - Tests for ensuring `$(MERLIN_SPEC_ORIGINAL_TEMPLATE)`, `$(MERLIN_SPEC_ARCHIVED_COPY)`, and `$(MERLIN_SPEC_EXECUTED_RUN)` are stored correctly diff --git a/merlin/study/batch.py b/merlin/study/batch.py index eeaead5ee..f5b62409f 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -92,7 +92,7 @@ def check_for_scheduler(scheduler, scheduler_legend): if result and len(result) > 0 and scheduler_legend[scheduler]["expected check output"] in result[0]: return True return False - except FileNotFoundError: + except (FileNotFoundError, PermissionError): return False From c01f6358402ee7c20a486560adf2e8f0591150b5 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Fri, 4 Aug 2023 16:04:11 -0700 Subject: [PATCH 11/29] Release/1.10.2 (#437) * bump version to 1.10.2 * bump version in CHANGELOG --- .gitignore | 1 + CHANGELOG.md | 2 +- Makefile | 2 +- merlin/__init__.py | 4 ++-- merlin/ascii_art.py | 2 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 2 +- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- merlin/common/security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 2 +- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/celeryconfig.py | 2 +- merlin/config/configfile.py | 2 +- merlin/config/results_backend.py | 2 +- merlin/config/utils.py | 2 +- merlin/data/celery/__init__.py | 2 +- merlin/display.py | 2 +- merlin/examples/__init__.py | 2 +- merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 2 +- merlin/merlin_templates.py | 2 +- merlin/router.py | 2 +- merlin/server/__init__.py | 2 +- merlin/server/server_commands.py | 2 +- merlin/server/server_config.py | 2 +- merlin/server/server_util.py | 2 +- merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 2 +- merlin/spec/expansion.py | 2 +- merlin/spec/override.py | 2 +- merlin/spec/specification.py | 2 +- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 2 +- merlin/study/celeryadapter.py | 2 +- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 2 +- merlin/study/study.py | 2 +- merlin/utils.py | 2 +- setup.py | 2 +- tests/integration/conditions.py | 2 +- tests/integration/run_tests.py | 2 +- tests/integration/test_definitions.py | 2 +- 56 files changed, 57 insertions(+), 56 deletions(-) diff --git a/.gitignore b/.gitignore index 8b0fb8ad2..c22521934 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,4 @@ jupyter/testDistributedSamples.py dist/ build/ .DS_Store +.vscode/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 981ce7198..5e7c0b1cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [unreleased] +## [1.10.2] ### Fixed - A bug where the .orig, .partial, and .expanded file names were using the study name rather than the original file name - A bug where the openfoam_wf_singularity example was not being found diff --git a/Makefile b/Makefile index 0153f10d4..030ed8d15 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/__init__.py b/merlin/__init__.py index 32bf5c875..9856d9a7b 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.10.1" +__version__ = "1.10.2" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index bb804d876..5d61d8ed1 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index 072c83b58..dc0cfd9d7 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index 1e9c75683..05c5d0bbf 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index 1e9c75683..05c5d0bbf 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index 81c8865e5..be663a572 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index f51055ab5..a4c127a12 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index 65d503564..f8b881dcf 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index 635dd617b..821c1df71 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index eb24e067e..fe9d735d4 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index 1e9c75683..05c5d0bbf 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index 22c505df4..dfd230a1a 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index 4d7dc176f..2c70fadc9 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index 3b41cb459..b24e0af3e 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index 43dbd133a..1f851abb2 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index aee1b62b0..a53190df7 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index 0cf9b7c6d..8c7dc8a2b 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/celeryconfig.py b/merlin/config/celeryconfig.py index 2740aa0a1..73605768e 100644 --- a/merlin/config/celeryconfig.py +++ b/merlin/config/celeryconfig.py @@ -10,7 +10,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index 6aa38f9b2..6ccf3dea5 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index 691bf5a6c..d30d5ab58 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/utils.py b/merlin/config/utils.py index 9778cde3c..2f5434091 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index 1e9c75683..05c5d0bbf 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index 89d14781e..bf60bb50d 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index 1e9c75683..05c5d0bbf 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index a66abf64a..a7754cec7 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index 308785784..5b893ab9e 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index cfbc21e38..b176378a9 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index 580952b1d..e90e13324 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index f49d3cc94..d3fe4b80a 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index 4809c7aee..b2bcc4949 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/router.py b/merlin/router.py index 160909ddf..779333dc6 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/__init__.py b/merlin/server/__init__.py index 7c6148fee..6cd46fa97 100644 --- a/merlin/server/__init__.py +++ b/merlin/server/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index d53463c2b..93f928f3b 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index 14abfb0d4..e088eedcd 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index d101af2f0..a3cc7f021 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index 1e9c75683..05c5d0bbf 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index f58731336..145eb00bb 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index b20ce7d09..1560f412d 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index 38f03d6e9..abba6ef02 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/override.py b/merlin/spec/override.py index 0f1e71f62..e9ec4bceb 100644 --- a/merlin/spec/override.py +++ b/merlin/spec/override.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 8d36297ab..1cb26d512 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index 1e9c75683..05c5d0bbf 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index f5b62409f..b2157ebde 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 4c01cfd73..f9c04b1e3 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index a85a86e47..d2315bf7f 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index 2df053051..dc01ad6a9 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index 14f273dfa..686b5afa2 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index 74b7c3181..8eaf306ca 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/utils.py b/merlin/utils.py index 7c18407e7..a2e4966d0 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/setup.py b/setup.py index 9693b06ba..0bb413325 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index 4da8c36a1..0bc687923 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 9b0270d3b..82a4fdd92 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index d20efb329..e9e9ce590 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.1. +# This file is part of Merlin, Version: 1.10.2. # # For details, see https://github.com/LLNL/merlin. # From 9c52ba2799e52f713f622ece8139f67a2d2b04d3 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Fri, 4 Aug 2023 17:04:11 -0700 Subject: [PATCH 12/29] resolve develop to main merge issues (#439) * fix default worker bug with all steps * version bump and requirements fix From b0f4d866e57b2edb324d5773a041122a4f1930d3 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Thu, 17 Aug 2023 12:37:08 -0700 Subject: [PATCH 13/29] dependabot/certifi-requests-pygments (#441) * Bump certifi from 2022.12.7 to 2023.7.22 in /docs Bumps [certifi](https://github.com/certifi/python-certifi) from 2022.12.7 to 2023.7.22. - [Commits](https://github.com/certifi/python-certifi/compare/2022.12.07...2023.07.22) --- updated-dependencies: - dependency-name: certifi dependency-type: direct:production ... Signed-off-by: dependabot[bot] * add all dependabot changes and update CHANGELOG --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- CHANGELOG.md | 10 ++++++++++ docs/requirements.txt | 6 +++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e7c0b1cd..b3ac21def 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [unreleased] +### Fixed + +### Added + +### Changed +- Bump certifi from 2022.12.7 to 2023.7.22 in /docs +- Bump pygments from 2.13.0 to 2.15.0 in /docs +- Bump requests from 2.28.1 to 2.31.0 in /docs + ## [1.10.2] ### Fixed - A bug where the .orig, .partial, and .expanded file names were using the study name rather than the original file name diff --git a/docs/requirements.txt b/docs/requirements.txt index 87333eb50..c771e60dc 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -8,7 +8,7 @@ alabaster==0.7.12 # via sphinx babel==2.10.3 # via sphinx -certifi==2022.12.7 +certifi==2023.7.22 # via requests charset-normalizer==2.1.1 # via requests @@ -26,13 +26,13 @@ markupsafe==2.1.1 # via jinja2 packaging==21.3 # via sphinx -pygments==2.13.0 +pygments==2.15.0 # via sphinx pyparsing==3.0.9 # via packaging pytz==2022.5 # via babel -requests==2.28.1 +requests==2.31.0 # via sphinx snowballstemmer==2.2.0 # via sphinx From c641c5c6495ebc0e789019b8ff80e5d0419ff597 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Fri, 18 Aug 2023 13:02:19 -0700 Subject: [PATCH 14/29] bugfix/server-pip-redis-conf (#443) * add *.conf to the MANIFEST file so pip will grab the redis.conf file * add note explaining how to fix a hanging merlin server start * modify CHANGELOG * add second export option to docs and fix typo --- CHANGELOG.md | 2 ++ MANIFEST.in | 2 +- docs/source/merlin_server.rst | 2 +- docs/source/server/commands.rst | 7 +++++++ 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3ac21def..f5e60ea9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed ### Added +- The *.conf regex for the recursive-include of the merlin server directory so that pip will add it to the wheel +- A note to the docs for how to fix an issue where the `merlin server start` command hangs ### Changed - Bump certifi from 2022.12.7 to 2023.7.22 in /docs diff --git a/MANIFEST.in b/MANIFEST.in index da9d411ad..d5526e37c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ recursive-include merlin/data *.yaml *.py -recursive-include merlin/server *.yaml *.py +recursive-include merlin/server *.yaml *.py *.conf recursive-include merlin/examples * include requirements.txt include requirements/* diff --git a/docs/source/merlin_server.rst b/docs/source/merlin_server.rst index 24b37c776..23f6d4a1d 100644 --- a/docs/source/merlin_server.rst +++ b/docs/source/merlin_server.rst @@ -1,7 +1,7 @@ Merlin Server ============= The merlin server command allows users easy access to containerized broker -and results servers for merlin workflows. This allowsusers to run merlin without +and results servers for merlin workflows. This allows users to run merlin without a dedicated external server. The main configuration will be stored in the subdirectory called "server/" by diff --git a/docs/source/server/commands.rst b/docs/source/server/commands.rst index dd8ca1b02..fc40bf182 100644 --- a/docs/source/server/commands.rst +++ b/docs/source/server/commands.rst @@ -34,6 +34,13 @@ Starting up a Merlin Server (``merlin server start``) Starts the container located in the local merlin server configuration. +.. note:: + If this command seems to hang and never release control back to you, follow these steps: + + 1. Kill the command with ``Ctrl+C`` + 2. Run either ``export LC_ALL="C.UTF-8"`` or ``export LC_ALL="C"`` + 3. Re-run the ``merlin server start`` command + Stopping an exisiting Merlin Server (``merlin server stop``) ------------------------------------------------------------ From 970a06fd6e4ba927b44180ad3bb4819df0fbdb73 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Fri, 18 Aug 2023 16:20:26 -0700 Subject: [PATCH 15/29] bump to version 1.10.3 (#444) --- CHANGELOG.md | 4 +--- Makefile | 2 +- merlin/__init__.py | 4 ++-- merlin/ascii_art.py | 2 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 2 +- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- merlin/common/security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 2 +- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/celeryconfig.py | 2 +- merlin/config/configfile.py | 2 +- merlin/config/results_backend.py | 2 +- merlin/config/utils.py | 2 +- merlin/data/celery/__init__.py | 2 +- merlin/display.py | 2 +- merlin/examples/__init__.py | 2 +- merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 2 +- merlin/merlin_templates.py | 2 +- merlin/router.py | 2 +- merlin/server/__init__.py | 2 +- merlin/server/server_commands.py | 2 +- merlin/server/server_config.py | 2 +- merlin/server/server_util.py | 2 +- merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 2 +- merlin/spec/expansion.py | 2 +- merlin/spec/override.py | 2 +- merlin/spec/specification.py | 2 +- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 2 +- merlin/study/celeryadapter.py | 2 +- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 2 +- merlin/study/study.py | 2 +- merlin/utils.py | 2 +- setup.py | 2 +- tests/integration/conditions.py | 2 +- tests/integration/run_tests.py | 2 +- tests/integration/test_definitions.py | 2 +- 55 files changed, 56 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5e60ea9d..c0760da46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,7 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [unreleased] -### Fixed - +## [1.10.3] ### Added - The *.conf regex for the recursive-include of the merlin server directory so that pip will add it to the wheel - A note to the docs for how to fix an issue where the `merlin server start` command hangs diff --git a/Makefile b/Makefile index 030ed8d15..74c407db0 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/__init__.py b/merlin/__init__.py index 9856d9a7b..12ba225cd 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.10.2" +__version__ = "1.10.3" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index 5d61d8ed1..b56da4d7a 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index dc0cfd9d7..9921bbb89 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index 05c5d0bbf..2a6208883 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index 05c5d0bbf..2a6208883 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index be663a572..a90133b73 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index a4c127a12..00aaea917 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index f8b881dcf..0f7607a8e 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index 821c1df71..4f7333f6d 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index fe9d735d4..55601073e 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index 05c5d0bbf..2a6208883 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index dfd230a1a..1059383d9 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index 2c70fadc9..cee757b91 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index b24e0af3e..56051756b 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index 1f851abb2..8137e0543 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index a53190df7..7af320b52 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index 8c7dc8a2b..78658333a 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/celeryconfig.py b/merlin/config/celeryconfig.py index 73605768e..e688945cc 100644 --- a/merlin/config/celeryconfig.py +++ b/merlin/config/celeryconfig.py @@ -10,7 +10,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index 6ccf3dea5..d46a1d038 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index d30d5ab58..a619ecb03 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/utils.py b/merlin/config/utils.py index 2f5434091..1385c4f35 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index 05c5d0bbf..2a6208883 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index bf60bb50d..f59255ddb 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index 05c5d0bbf..2a6208883 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index a7754cec7..78152e6ee 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index 5b893ab9e..5dbe2ebf5 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index b176378a9..ed7156aa5 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index e90e13324..6a6da63d8 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index d3fe4b80a..a29546798 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index b2bcc4949..477887794 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/router.py b/merlin/router.py index 779333dc6..465f0ad3d 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/__init__.py b/merlin/server/__init__.py index 6cd46fa97..88f37fd2c 100644 --- a/merlin/server/__init__.py +++ b/merlin/server/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index 93f928f3b..45e6ef3d3 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index e088eedcd..5f109f7c8 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index a3cc7f021..a280abac5 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index 05c5d0bbf..2a6208883 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index 145eb00bb..7f9f66188 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index 1560f412d..c4aad952c 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index abba6ef02..e29f0e7b5 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/override.py b/merlin/spec/override.py index e9ec4bceb..50d4f1c35 100644 --- a/merlin/spec/override.py +++ b/merlin/spec/override.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 1cb26d512..eb165b617 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index 05c5d0bbf..2a6208883 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index b2157ebde..4298fca32 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index f9c04b1e3..31ef03b7c 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index d2315bf7f..6c977a756 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index dc01ad6a9..be66e1b97 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index 686b5afa2..bdba0250d 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index 8eaf306ca..8f4ddb19d 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/utils.py b/merlin/utils.py index a2e4966d0..3eb4e5acc 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/setup.py b/setup.py index 0bb413325..d409c0a74 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index 0bc687923..81c063112 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 82a4fdd92..bc19fd9c8 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index e9e9ce590..0fdedf07b 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.2. +# This file is part of Merlin, Version: 1.10.3. # # For details, see https://github.com/LLNL/merlin. # From d8bfbdbd5a07a314064ce24a6201662a34fc8028 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Tue, 22 Aug 2023 12:54:18 -0700 Subject: [PATCH 16/29] bugfix/sphinx-5.3.0-requirement (#446) * Version/1.10.3 (#445) * fix default worker bug with all steps * version bump and requirements fix * Bugfix/filename-special-vars (#425) * fix file naming bug * fix filename bug with variable as study name * add tests for the file name special vars changes * modify changelog * implement Luc's suggestions * remove replace line * Create dependabot-changelog-updater.yml * testing outputs of modifying changelog * delete dependabot-changelog-updater * feature/pdf-docs (#427) * first attempt at adding pdf * fixing build error * modify changelog to show docs changes * fix errors Luc found in the build logs * trying out removal of latex * reverting latex changes back * uncommenting the latex_elements settings * adding epub to see if latex will build * adding a latex engine variable to conf * fix naming error with latex_engine * attempting to add a logo to the pdf build * testing an override to the searchtools file * revert back to not using searchtools override * update changelog * bugfix/openfoam_singularity_issues (#426) * fix openfoam_singularity issues * update requirements and descriptions for openfoam examples * bugfix/output-path-substitution (#430) * fix bug with output_path and variable substitution * add tests for cli substitutions * bugfix/scheduler-permission-error (#436) * Release/1.10.2 (#437) * bump version to 1.10.2 * bump version in CHANGELOG * resolve develop to main merge issues (#439) * fix default worker bug with all steps * version bump and requirements fix * dependabot/certifi-requests-pygments (#441) * Bump certifi from 2022.12.7 to 2023.7.22 in /docs Bumps [certifi](https://github.com/certifi/python-certifi) from 2022.12.7 to 2023.7.22. - [Commits](https://github.com/certifi/python-certifi/compare/2022.12.07...2023.07.22) --- updated-dependencies: - dependency-name: certifi dependency-type: direct:production ... Signed-off-by: dependabot[bot] * add all dependabot changes and update CHANGELOG --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * bugfix/server-pip-redis-conf (#443) * add *.conf to the MANIFEST file so pip will grab the redis.conf file * add note explaining how to fix a hanging merlin server start * modify CHANGELOG * add second export option to docs and fix typo * bump to version 1.10.3 (#444) --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * change hardcoded sphinx requirement * update CHANGELOG --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- CHANGELOG.md | 4 +++ docs/requirements.in | 4 +++ docs/requirements.txt | 57 +------------------------------------------ 3 files changed, 9 insertions(+), 56 deletions(-) create mode 100644 docs/requirements.in diff --git a/CHANGELOG.md b/CHANGELOG.md index c0760da46..beecf82f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [unreleased] +### Changed +- Hardcoded Sphinx v5.3.0 requirement is now removed so we can use latest Sphinx + ## [1.10.3] ### Added - The *.conf regex for the recursive-include of the merlin server directory so that pip will add it to the wheel diff --git a/docs/requirements.in b/docs/requirements.in new file mode 100644 index 000000000..268785121 --- /dev/null +++ b/docs/requirements.in @@ -0,0 +1,4 @@ +# This file will list all requirements for the docs so we can freeze a version of them for release. +# To freeze the versions run: +# pip-compile requirements.in +sphinx \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt index c771e60dc..5d3faecfe 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,56 +1 @@ -# -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: -# -# pip-compile requirements.in -# -alabaster==0.7.12 - # via sphinx -babel==2.10.3 - # via sphinx -certifi==2023.7.22 - # via requests -charset-normalizer==2.1.1 - # via requests -docutils==0.17.1 - # via sphinx -idna==3.4 - # via requests -imagesize==1.4.1 - # via sphinx -importlib-metadata==5.0.0 - # via sphinx -jinja2==3.0.3 - # via sphinx -markupsafe==2.1.1 - # via jinja2 -packaging==21.3 - # via sphinx -pygments==2.15.0 - # via sphinx -pyparsing==3.0.9 - # via packaging -pytz==2022.5 - # via babel -requests==2.31.0 - # via sphinx -snowballstemmer==2.2.0 - # via sphinx -sphinx==5.3.0 - # via -r requirements.in -sphinxcontrib-applehelp==1.0.2 - # via sphinx -sphinxcontrib-devhelp==1.0.2 - # via sphinx -sphinxcontrib-htmlhelp==2.0.0 - # via sphinx -sphinxcontrib-jsmath==1.0.1 - # via sphinx -sphinxcontrib-qthelp==1.0.3 - # via sphinx -sphinxcontrib-serializinghtml==1.1.5 - # via sphinx -urllib3==1.26.12 - # via requests -zipp==3.10.0 - # via importlib-metadata +sphinx>=5.3.0 From 8241bfe21d5eb58fd2f9f8a3235181c0ea27faaf Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Wed, 27 Sep 2023 17:19:59 -0700 Subject: [PATCH 17/29] feature/vlauncher (#447) * fix file naming error for iterative workflows * fixed small bug with new filepath naming * add VLAUNCHER functionality * add docs for VLAUNCHER and modify changelog * re-word docs and fix table format * add a test for vlauncher * run fix-style and add a test for vlauncher * Add the find_vlaunch_var and setup_vlaunch functions. The numeric value of the shell variables may not be defined until run time, so replace with variable strings instead of values. Consolidate the commands into one function. * Add variable set for (t)csh. * Run fix-style * make step settings the defaults and ignore commented lines * add some additional tests * remove regex library import --------- Co-authored-by: Joseph M. Koning --- CHANGELOG.md | 7 ++ docs/source/merlin_variables.rst | 53 ++++++++++++- merlin/spec/defaults.py | 8 ++ merlin/spec/specification.py | 13 +++- merlin/study/script_adapter.py | 77 ++++++++++++++++++- merlin/study/study.py | 3 +- merlin/utils.py | 20 +++++ tests/integration/test_definitions.py | 82 +++++++++++++++++++-- tests/integration/test_specs/flux_test.yaml | 28 +++++++ 9 files changed, 279 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index beecf82f7..135508a8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,9 +5,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [unreleased] +### Added +- New reserved variables: + - `VLAUNCHER`: The same functionality as the `LAUNCHER` variable, but will substitute shell variables `MERLIN_NODES`, `MERLIN_PROCS`, `MERLIN_CORES`, and `MERLIN_GPUS` for nodes, procs, cores per task, and gpus + ### Changed - Hardcoded Sphinx v5.3.0 requirement is now removed so we can use latest Sphinx +### Fixed +- A bug where the filenames in iterative workflows kept appending `.out`, `.partial`, or `.expanded` to the filenames stored in the `merlin_info/` subdirectory + ## [1.10.3] ### Added - The *.conf regex for the recursive-include of the merlin server directory so that pip will add it to the wheel diff --git a/docs/source/merlin_variables.rst b/docs/source/merlin_variables.rst index d67e06412..f8ea7fce7 100644 --- a/docs/source/merlin_variables.rst +++ b/docs/source/merlin_variables.rst @@ -160,8 +160,9 @@ Reserved variables $(MERLIN_INFO)/*.expanded.yaml -The ``LAUNCHER`` Variable -+++++++++++++++++++++++++ + +The ``LAUNCHER`` and ``VLAUNCHER`` Variables ++++++++++++++++++++++++++++++++++++++++++++++++ ``$(LAUNCHER)`` is a special case of a reserved variable since it's value *can* be changed. It serves as an abstraction to launch a job with parallel schedulers like :ref:`slurm`, @@ -187,6 +188,54 @@ We can modify this to use the ``$(LAUNCHER)`` variable like so: In other words, the ``$(LAUNCHER)`` variable would become ``srun -N 1 -n 3``. +Similarly, the ``$(VLAUNCHER)`` variable behaves similarly to the ``$(LAUNCHER)`` variable. +The key distinction lies in its source of information. Instead of drawing certain configuration +options from the ``run`` section of a step, it retrieves specific shell variables. These shell +variables are automatically generated by Merlin when you include the ``$(VLAUNCHER)`` variable +in a step command, but they can also be customized by the user. Currently, the following shell +variables are: + +.. list-table:: VLAUNCHER Variables + :widths: 25 50 25 + :header-rows: 1 + + * - Variable + - Description + - Default + + * - ``${MERLIN_NODES}`` + - The number of nodes + - 1 + + * - ``${MERLIN_PROCS}`` + - The number of tasks/procs + - 1 + + * - ``${MERLIN_CORES}`` + - The number of cores per task/proc + - 1 + + * - ``${MERLIN_GPUS}`` + - The number of gpus per task/proc + - 0 + +Let's say we have the following defined in our yaml file: + +.. code:: yaml + + batch: + type: flux + + run: + cmd: | + MERLIN_NODES=4 + MERLIN_PROCS=2 + MERLIN_CORES=8 + MERLIN_GPUS=2 + $(VLAUNCHER) python script.py + +The ``$(VLAUNCHER)`` variable would be substituted to ``flux run -N 4 -n 2 -c 8 -g 2``. + User variables ------------------- Variables defined by a specification file in the ``env`` section, as in this example: diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index c4aad952c..6845b8b08 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -52,3 +52,11 @@ "generate": {"cmd": "echo 'Insert sample-generating command here'"}, "level_max_dirs": 25, } + +# Values of the form (step key to search for, default value if no step key found) +VLAUNCHER_VARS = { + "MERLIN_NODES": ("nodes", 1), + "MERLIN_PROCS": ("procs", 1), + "MERLIN_CORES": ("cores per task", 1), + "MERLIN_GPUS": ("gpus", 0), +} diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index eb165b617..60a47964d 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -45,7 +45,7 @@ from maestrowf.specification import YAMLSpecification from merlin.spec import all_keys, defaults -from merlin.utils import repr_timedelta +from merlin.utils import find_vlaunch_var, repr_timedelta LOG = logging.getLogger(__name__) @@ -369,6 +369,17 @@ def process_spec_defaults(self): defaults.STUDY_STEP_RUN["shell"] = self.batch["shell"] for step in self.study: MerlinSpec.fill_missing_defaults(step["run"], defaults.STUDY_STEP_RUN) + # Insert VLAUNCHER specific variables if necessary + if "$(VLAUNCHER)" in step["run"]["cmd"]: + SHSET = "" + if "csh" in step["run"]["shell"]: + SHSET = "set " + # We need to set default values for VLAUNCHER variables if they're not defined by the user + for vlaunch_var, vlaunch_val in defaults.VLAUNCHER_VARS.items(): + if not find_vlaunch_var(vlaunch_var.replace("MERLIN_", ""), step["run"]["cmd"], accept_no_matches=True): + # Look for predefined nodes/procs/cores/gpus values in the step and default to those + vlaunch_val = step["run"][vlaunch_val[0]] if vlaunch_val[0] in step["run"] else vlaunch_val[1] + step["run"]["cmd"] = f"{SHSET}{vlaunch_var}={vlaunch_val}\n" + step["run"]["cmd"] # fill in missing merlin section defaults MerlinSpec.fill_missing_defaults(self.merlin, defaults.MERLIN["merlin"]) diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index be66e1b97..80e56f279 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -42,12 +42,36 @@ from maestrowf.utils import start_process from merlin.common.abstracts.enums import ReturnCode -from merlin.utils import convert_timestring +from merlin.utils import convert_timestring, find_vlaunch_var LOG = logging.getLogger(__name__) +def setup_vlaunch(step_run: str, batch_type: str, gpu_config: bool) -> None: + """ + Check for the VLAUNCHER keyword int the step run string, find + the MERLIN variables and configure VLAUNCHER. + + :param `step_run`: the step.run command string + :param `batch_type`: the batch type string + :param `gpu_config`: bool to determin if gpus should be configured + :returns: None + """ + if "$(VLAUNCHER)" in step_run["cmd"]: + step_run["cmd"] = step_run["cmd"].replace("$(VLAUNCHER)", "$(LAUNCHER)") + + step_run["nodes"] = find_vlaunch_var("NODES", step_run["cmd"]) + step_run["procs"] = find_vlaunch_var("PROCS", step_run["cmd"]) + step_run["cores per task"] = find_vlaunch_var("CORES", step_run["cmd"]) + + if find_vlaunch_var("GPUS", step_run["cmd"]): + if gpu_config: + step_run["gpus"] = find_vlaunch_var("GPUS", step_run["cmd"]) + else: + LOG.warning(f"Merlin does not yet have the ability to set GPUs per task with {batch_type}. Coming soon.") + + class MerlinLSFScriptAdapter(SlurmScriptAdapter): """ A SchedulerScriptAdapter class for slurm blocking parallel launches, @@ -156,6 +180,23 @@ def get_parallelize_command(self, procs, nodes=None, **kwargs): return " ".join(args) + def write_script(self, ws_path, step): + """ + This will overwrite the write_script in method from Maestro's base ScriptAdapter + class but will eventually call it. This is necessary for the VLAUNCHER to work. + + :param `ws_path`: the path to the workspace where we'll write the scripts + :param `step`: the Maestro StudyStep object containing info for our step + :returns: a tuple containing: + - a boolean representing whether this step is to be scheduled or not + - Merlin can ignore this + - a path to the script for the cmd + - a path to the script for the restart cmd + """ + setup_vlaunch(step.run, "lsf", False) + + return super().write_script(ws_path, step) + class MerlinSlurmScriptAdapter(SlurmScriptAdapter): """ @@ -256,6 +297,23 @@ def get_parallelize_command(self, procs, nodes=None, **kwargs): return " ".join(args) + def write_script(self, ws_path, step): + """ + This will overwrite the write_script in method from Maestro's base ScriptAdapter + class but will eventually call it. This is necessary for the VLAUNCHER to work. + + :param `ws_path`: the path to the workspace where we'll write the scripts + :param `step`: the Maestro StudyStep object containing info for our step + :returns: a tuple containing: + - a boolean representing whether this step is to be scheduled or not + - Merlin can ignore this + - a path to the script for the cmd + - a path to the script for the restart cmd + """ + setup_vlaunch(step.run, "slurm", False) + + return super().write_script(ws_path, step) + class MerlinFluxScriptAdapter(MerlinSlurmScriptAdapter): """ @@ -319,6 +377,23 @@ def time_format(self, val): """ return convert_timestring(val, format_method="FSD") + def write_script(self, ws_path, step): + """ + This will overwrite the write_script in method from Maestro's base ScriptAdapter + class but will eventually call it. This is necessary for the VLAUNCHER to work. + + :param `ws_path`: the path to the workspace where we'll write the scripts + :param `step`: the Maestro StudyStep object containing info for our step + :returns: a tuple containing: + - a boolean representing whether this step is to be scheduled or not + - Merlin can ignore this + - a path to the script for the cmd + - a path to the script for the restart cmd + """ + setup_vlaunch(step.run, "flux", True) + + return super().write_script(ws_path, step) + class MerlinScriptAdapter(LocalScriptAdapter): """ diff --git a/merlin/study/study.py b/merlin/study/study.py index 8f4ddb19d..831062e17 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -127,7 +127,8 @@ def __init__( # pylint: disable=R0913 def _set_special_file_vars(self): """Setter for the orig, partial, and expanded file paths of a study.""" - base_name = Path(self.filepath).stem + shortened_filepath = self.filepath.replace(".out", "").replace(".partial", "").replace(".expanded", "") + base_name = Path(shortened_filepath).stem self.special_vars["MERLIN_SPEC_ORIGINAL_TEMPLATE"] = os.path.join( self.info, base_name + ".orig.yaml", diff --git a/merlin/utils.py b/merlin/utils.py index 3eb4e5acc..51a1fd8c0 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -497,6 +497,26 @@ def contains_shell_ref(string): return False +def find_vlaunch_var(vlaunch_var: str, step_cmd: str, accept_no_matches=False) -> str: + """ + Given a variable used for VLAUNCHER and the step cmd value, find + the variable. + + :param `vlaunch_var`: The name of the VLAUNCHER variable (without MERLIN_) + :param `step_cmd`: The string for the cmd of a step + :param `accept_no_matches`: If True, return None if we couldn't find the variable. Otherwise, raise an error. + :returns: the `vlaunch_var` variable or None + """ + matches = list(re.findall(rf"^(?!#).*MERLIN_{vlaunch_var}=\d+", step_cmd, re.MULTILINE)) + + if matches: + return f"${{MERLIN_{vlaunch_var}}}" + + if accept_no_matches: + return None + raise ValueError(f"VLAUNCHER used but could not find MERLIN_{vlaunch_var} in the step.") + + # Time utilities def convert_to_timedelta(timestr: Union[str, int]) -> timedelta: """Convert a timestring to a timedelta object. diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index 0fdedf07b..2f37dcc95 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -407,13 +407,81 @@ def define_tests(): # pylint: disable=R0914,R0915 }, "dry launch flux": { "cmds": f"{run} {flux} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", - "conditions": StepFileHasRegex( - "runs", - "*/runs.slurm.sh", - "flux_test", - OUTPUT_DIR, - get_flux_cmd("flux", no_errors=True), - ), + "conditions": [ + StepFileHasRegex( + "runs", + "*/runs.slurm.sh", + "flux_test", + OUTPUT_DIR, + get_flux_cmd("flux", no_errors=True), + ), + ################## + # VLAUNCHER TESTS + ################## + StepFileHasRegex( + "vlauncher_test", + "vlauncher_test.slurm.sh", + "flux_test", + OUTPUT_DIR, + r"flux run -n \$\{MERLIN_PROCS\} -N \$\{MERLIN_NODES\} -c \$\{MERLIN_CORES\}", + ), + StepFileHasRegex( + "vlauncher_test_step_defaults", + "vlauncher_test_step_defaults.slurm.sh", + "flux_test", + OUTPUT_DIR, + r"MERLIN_GPUS=1", + ), + StepFileHasRegex( + "vlauncher_test_step_defaults", + "vlauncher_test_step_defaults.slurm.sh", + "flux_test", + OUTPUT_DIR, + r"MERLIN_NODES=6", + ), + StepFileHasRegex( + "vlauncher_test_step_defaults", + "vlauncher_test_step_defaults.slurm.sh", + "flux_test", + OUTPUT_DIR, + r"MERLIN_PROCS=3", + ), + StepFileHasRegex( + "vlauncher_test_step_defaults", + "vlauncher_test_step_defaults.slurm.sh", + "flux_test", + OUTPUT_DIR, + r"MERLIN_CORES=2", + ), + StepFileHasRegex( + "vlauncher_test_no_step_defaults", + "vlauncher_test_no_step_defaults.slurm.sh", + "flux_test", + OUTPUT_DIR, + r"MERLIN_GPUS=0", + ), + StepFileHasRegex( + "vlauncher_test_no_step_defaults", + "vlauncher_test_no_step_defaults.slurm.sh", + "flux_test", + OUTPUT_DIR, + r"MERLIN_NODES=1", + ), + StepFileHasRegex( + "vlauncher_test_no_step_defaults", + "vlauncher_test_no_step_defaults.slurm.sh", + "flux_test", + OUTPUT_DIR, + r"MERLIN_PROCS=1", + ), + StepFileHasRegex( + "vlauncher_test_no_step_defaults", + "vlauncher_test_no_step_defaults.slurm.sh", + "flux_test", + OUTPUT_DIR, + r"MERLIN_CORES=1", + ), + ], "run type": "local", }, "dry launch lsf": { diff --git a/tests/integration/test_specs/flux_test.yaml b/tests/integration/test_specs/flux_test.yaml index fe0130526..99f15205c 100644 --- a/tests/integration/test_specs/flux_test.yaml +++ b/tests/integration/test_specs/flux_test.yaml @@ -33,6 +33,34 @@ study: depends: [runs*] task_queue: flux_test +- description: step that uses vlauncher + name: vlauncher_test + run: + cmd: | + MERLIN_NODES=6 + MERLIN_PROCS=3 + MERLIN_CORES=2 + $(VLAUNCHER) echo "step that uses vlauncher" + task_queue: flux_test + +- description: test vlauncher step defaults + name: vlauncher_test_step_defaults + run: + cmd: | + $(VLAUNCHER) echo "test vlauncher step defaults" + task_queue: flux_test + nodes: 6 + procs: 3 + cores per task: 2 + gpus: 1 + +- description: test vlauncher no step defaults + name: vlauncher_test_no_step_defaults + run: + cmd: | + $(VLAUNCHER) echo "test vlauncher no step defaults" + task_queue: flux_test + global.parameters: STUDY: label: STUDY.%% From 50d0fb6a906146cf40fe06fb15edf0f85f87670b Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Thu, 28 Sep 2023 08:14:38 -0700 Subject: [PATCH 18/29] release/1.11.0 (#448) --- CHANGELOG.md | 4 ++-- Makefile | 2 +- merlin/__init__.py | 4 ++-- merlin/ascii_art.py | 2 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 2 +- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- merlin/common/security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 2 +- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/celeryconfig.py | 2 +- merlin/config/configfile.py | 2 +- merlin/config/results_backend.py | 2 +- merlin/config/utils.py | 2 +- merlin/data/celery/__init__.py | 2 +- merlin/display.py | 2 +- merlin/examples/__init__.py | 2 +- merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 2 +- merlin/merlin_templates.py | 2 +- merlin/router.py | 2 +- merlin/server/__init__.py | 2 +- merlin/server/server_commands.py | 2 +- merlin/server/server_config.py | 2 +- merlin/server/server_util.py | 2 +- merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 2 +- merlin/spec/expansion.py | 2 +- merlin/spec/override.py | 2 +- merlin/spec/specification.py | 2 +- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 2 +- merlin/study/celeryadapter.py | 2 +- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 2 +- merlin/study/study.py | 2 +- merlin/utils.py | 2 +- setup.py | 2 +- tests/integration/conditions.py | 2 +- tests/integration/run_tests.py | 2 +- tests/integration/test_definitions.py | 2 +- 55 files changed, 57 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 135508a8c..914d8616a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,9 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [unreleased] +## [1.11.0] ### Added -- New reserved variables: +- New reserved variable: - `VLAUNCHER`: The same functionality as the `LAUNCHER` variable, but will substitute shell variables `MERLIN_NODES`, `MERLIN_PROCS`, `MERLIN_CORES`, and `MERLIN_GPUS` for nodes, procs, cores per task, and gpus ### Changed diff --git a/Makefile b/Makefile index 74c407db0..4a857a217 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/__init__.py b/merlin/__init__.py index 12ba225cd..20a0e8b3e 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.10.3" +__version__ = "1.11.0" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index b56da4d7a..f823937a6 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index 9921bbb89..95f26530e 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index 2a6208883..d6f53d03d 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index 2a6208883..d6f53d03d 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index a90133b73..7b8ab80f5 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index 00aaea917..124c7851d 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index 0f7607a8e..a8f8dffb2 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index 4f7333f6d..149d52e13 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index 55601073e..dc13d41d1 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index 2a6208883..d6f53d03d 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index 1059383d9..125ec5bed 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index cee757b91..68e178b77 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index 56051756b..f1d06077a 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index 8137e0543..c29763485 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index 7af320b52..0594ffe45 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index 78658333a..fe49ff162 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/celeryconfig.py b/merlin/config/celeryconfig.py index e688945cc..0ff305962 100644 --- a/merlin/config/celeryconfig.py +++ b/merlin/config/celeryconfig.py @@ -10,7 +10,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index d46a1d038..1f3418377 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index a619ecb03..d3e7002e7 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/utils.py b/merlin/config/utils.py index 1385c4f35..65fc6f85c 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index 2a6208883..d6f53d03d 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index f59255ddb..a0470938c 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index 2a6208883..d6f53d03d 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index 78152e6ee..1d756f00e 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index 5dbe2ebf5..294787857 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index ed7156aa5..cf272d93b 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index 6a6da63d8..3fba8cfc8 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index a29546798..198cf3804 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index 477887794..7936db03b 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/router.py b/merlin/router.py index 465f0ad3d..476ab1c0f 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/__init__.py b/merlin/server/__init__.py index 88f37fd2c..d04c75d72 100644 --- a/merlin/server/__init__.py +++ b/merlin/server/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index 45e6ef3d3..45411131b 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index 5f109f7c8..414f7a407 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index a280abac5..2b8f1216d 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index 2a6208883..d6f53d03d 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index 7f9f66188..556f5924e 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index 6845b8b08..8972d5cfe 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index e29f0e7b5..381bc72f4 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/override.py b/merlin/spec/override.py index 50d4f1c35..f3192a38e 100644 --- a/merlin/spec/override.py +++ b/merlin/spec/override.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 60a47964d..32fe0f635 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index 2a6208883..d6f53d03d 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index 4298fca32..e02a65a32 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 31ef03b7c..8b5ff196d 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index 6c977a756..ea4d22926 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index 80e56f279..6ecc79c5f 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index bdba0250d..5d877ba4f 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index 831062e17..b9ada35ea 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/utils.py b/merlin/utils.py index 51a1fd8c0..33735085d 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/setup.py b/setup.py index d409c0a74..7c91d26c7 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index 81c063112..b25010ca2 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index bc19fd9c8..58460e18f 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index 2f37dcc95..f59acf237 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.10.3. +# This file is part of Merlin, Version: 1.11.0. # # For details, see https://github.com/LLNL/merlin. # From 99d56598bcd2ed90743f5a0470f604fabfbc6704 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Mon, 9 Oct 2023 13:46:23 -0700 Subject: [PATCH 19/29] bugfix/skewed-sample-hierarchy (#450) * add patch for skewed sample hierarchy/additional samples * update changelog * catch narrower range of exceptions --- CHANGELOG.md | 1 + merlin/common/tasks.py | 48 ++++++++++++++++++++++++------------------ 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 914d8616a..f994a4d05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - A bug where the filenames in iterative workflows kept appending `.out`, `.partial`, or `.expanded` to the filenames stored in the `merlin_info/` subdirectory +- A bug where a skewed sample hierarchy was created when a restart was necessary in the `add_merlin_expanded_chain_to_chord` task ## [1.10.3] ### Added diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index f1d06077a..fbd401826 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -298,27 +298,33 @@ def add_merlin_expanded_chain_to_chord( # pylint: disable=R0913,R0914 LOG.debug("chain added to chord") else: # recurse down the sample_index hierarchy - LOG.debug("recursing down sample_index hierarchy") - for next_index in sample_index.children.values(): - next_index.name = os.path.join(sample_index.name, next_index.name) - LOG.debug("generating next step") - next_step = add_merlin_expanded_chain_to_chord.s( - task_type, - chain_, - samples[next_index.min - min_sample_id : next_index.max - min_sample_id], - labels, - next_index, - adapter_config, - next_index.min, - ) - next_step.set(queue=chain_[0].get_task_queue()) - LOG.debug(f"recursing with range {next_index.min}:{next_index.max}, {next_index.name} {signature(next_step)}") - LOG.debug(f"queuing samples[{next_index.min}:{next_index.max}] in for {chain_} in {next_index.name}...") - if self.request.is_eager: - next_step.delay() - else: - self.add_to_chord(next_step, lazy=False) - LOG.debug(f"queued for samples[{next_index.min}:{next_index.max}] in for {chain_} in {next_index.name}") + try: + LOG.debug("recursing down sample_index hierarchy") + for next_index in sample_index.children.values(): + next_index_name_before = next_index.name + next_index.name = os.path.join(sample_index.name, next_index.name) + LOG.debug("generating next step") + next_step = add_merlin_expanded_chain_to_chord.s( + task_type, + chain_, + samples[next_index.min - min_sample_id : next_index.max - min_sample_id], + labels, + next_index, + adapter_config, + next_index.min, + ) + next_step.set(queue=chain_[0].get_task_queue()) + LOG.debug(f"recursing with range {next_index.min}:{next_index.max}, {next_index.name} {signature(next_step)}") + LOG.debug(f"queuing samples[{next_index.min}:{next_index.max}] in for {chain_} in {next_index.name}...") + if self.request.is_eager: + next_step.delay() + else: + self.add_to_chord(next_step, lazy=False) + LOG.debug(f"queued for samples[{next_index.min}:{next_index.max}] in for {chain_} in {next_index.name}") + except retry_exceptions as e: + # Reset the index to what it was before so we don't accidentally create a bunch of extra samples upon restart + next_index.name = next_index_name_before + raise e return ReturnCode.OK From 593dbcdf90096b722a97da8abe62e23c016f18e5 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Mon, 23 Oct 2023 10:54:24 -0700 Subject: [PATCH 20/29] bugfix/lsf-gpu-typo (#453) * fix typo in batch.py that causes a bug * change print statements to log statements --- CHANGELOG.md | 4 ++++ merlin/study/batch.py | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f994a4d05..d709ccaa4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] +### Fixed +- Typo in `batch.py` that caused lsf launches to fail (`ALL_SGPUS` changed to `ALL_GPUS`) + ## [1.11.0] ### Added - New reserved variable: diff --git a/merlin/study/batch.py b/merlin/study/batch.py index e02a65a32..1b96cd282 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -299,7 +299,7 @@ def construct_scheduler_legend(parsed_batch: Dict, nodes: int) -> Dict: "lsf": { "check cmd": ["jsrun", "--help"], "expected check output": b"jsrun", - "launch": f"jsrun -a 1 -c ALL_CPUS -g ALL_SGPUS --bind=none -n {nodes}", + "launch": f"jsrun -a 1 -c ALL_CPUS -g ALL_GPUS --bind=none -n {nodes}", }, # pbs is mainly a placeholder in case a user wants to try it (we don't have it at the lab so it's mostly untested) "pbs": { @@ -335,12 +335,16 @@ def construct_worker_launch_command(parsed_batch: Dict, nodes: int) -> str: scheduler_legend: Dict = construct_scheduler_legend(parsed_batch, nodes) workload_manager: str = get_batch_type(scheduler_legend) + LOG.debug(f"parsed_batch: {parsed_batch}") + if parsed_batch["btype"] == "pbs" and workload_manager == parsed_batch["btype"]: raise TypeError("The PBS scheduler is only enabled for 'batch: flux' type") if parsed_batch["btype"] == "slurm" and workload_manager not in ("lsf", "flux", "pbs"): workload_manager = "slurm" + LOG.debug(f"workload_manager: {workload_manager}") + try: launch_command = scheduler_legend[workload_manager]["launch"] except KeyError as e: # pylint: disable=C0103 From f994f96f71d3405f59e108e0e67c9d2cd0ff3998 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Mon, 23 Oct 2023 11:08:53 -0700 Subject: [PATCH 21/29] release/1.11.1 (#454) --- CHANGELOG.md | 2 +- Makefile | 2 +- merlin/__init__.py | 4 ++-- merlin/ascii_art.py | 2 +- merlin/celery.py | 2 +- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 2 +- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- merlin/common/security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 2 +- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/celeryconfig.py | 2 +- merlin/config/configfile.py | 2 +- merlin/config/results_backend.py | 2 +- merlin/config/utils.py | 2 +- merlin/data/celery/__init__.py | 2 +- merlin/display.py | 2 +- merlin/examples/__init__.py | 2 +- merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 2 +- merlin/merlin_templates.py | 2 +- merlin/router.py | 2 +- merlin/server/__init__.py | 2 +- merlin/server/server_commands.py | 2 +- merlin/server/server_config.py | 2 +- merlin/server/server_util.py | 2 +- merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 2 +- merlin/spec/expansion.py | 2 +- merlin/spec/override.py | 2 +- merlin/spec/specification.py | 2 +- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 2 +- merlin/study/celeryadapter.py | 2 +- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 2 +- merlin/study/study.py | 2 +- merlin/utils.py | 2 +- setup.py | 2 +- tests/integration/conditions.py | 2 +- tests/integration/run_tests.py | 2 +- tests/integration/test_definitions.py | 2 +- 55 files changed, 56 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d709ccaa4..8d0ef2ae6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [1.11.1] ### Fixed - Typo in `batch.py` that caused lsf launches to fail (`ALL_SGPUS` changed to `ALL_GPUS`) diff --git a/Makefile b/Makefile index 4a857a217..2f9db031b 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/__init__.py b/merlin/__init__.py index 20a0e8b3e..c1ad21b22 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.11.0" +__version__ = "1.11.1" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index f823937a6..3cca2c710 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index 95f26530e..55d616658 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index d6f53d03d..e6dccdf56 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index d6f53d03d..e6dccdf56 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index 7b8ab80f5..383e7dccd 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index 124c7851d..d79e4e4f3 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index a8f8dffb2..da366b452 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index 149d52e13..4e3ac3a52 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index dc13d41d1..4303c3a6e 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index d6f53d03d..e6dccdf56 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index 125ec5bed..806d42e0c 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index 68e178b77..e0957ebb8 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index fbd401826..2bd77d2ad 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index c29763485..134d0b66c 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index 0594ffe45..b58e3b2a9 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index fe49ff162..385b8c1df 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/celeryconfig.py b/merlin/config/celeryconfig.py index 0ff305962..fbbd39064 100644 --- a/merlin/config/celeryconfig.py +++ b/merlin/config/celeryconfig.py @@ -10,7 +10,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index 1f3418377..2ca6c5d04 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index d3e7002e7..b88655399 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/utils.py b/merlin/config/utils.py index 65fc6f85c..8f0c6b029 100644 --- a/merlin/config/utils.py +++ b/merlin/config/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index d6f53d03d..e6dccdf56 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/display.py b/merlin/display.py index a0470938c..78eee5866 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index d6f53d03d..e6dccdf56 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index 1d756f00e..9b65f31ae 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index 294787857..2fa5e61ce 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index cf272d93b..72d5a1521 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index 3fba8cfc8..b8858f721 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index 198cf3804..55496a72c 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index 7936db03b..a355c4f2f 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/router.py b/merlin/router.py index 476ab1c0f..01a10aae7 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/__init__.py b/merlin/server/__init__.py index d04c75d72..7e2b6f1c1 100644 --- a/merlin/server/__init__.py +++ b/merlin/server/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index 45411131b..a28776577 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -8,7 +8,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_config.py b/merlin/server/server_config.py index 414f7a407..e4ec646fc 100644 --- a/merlin/server/server_config.py +++ b/merlin/server/server_config.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index 2b8f1216d..bab641702 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index d6f53d03d..e6dccdf56 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index 556f5924e..dcc02b063 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index 8972d5cfe..32fd05aa5 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index 381bc72f4..5924d1f74 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/override.py b/merlin/spec/override.py index f3192a38e..abb0f13c9 100644 --- a/merlin/spec/override.py +++ b/merlin/spec/override.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 32fe0f635..ac23b06d1 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index d6f53d03d..e6dccdf56 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index 1b96cd282..01a2945e3 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index 8b5ff196d..a6707d952 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/dag.py b/merlin/study/dag.py index ea4d22926..4cffc679c 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index 6ecc79c5f..45d211742 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index 5d877ba4f..c5773520a 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index b9ada35ea..6e2f4f937 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/utils.py b/merlin/utils.py index 33735085d..196e8b29b 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/setup.py b/setup.py index 7c91d26c7..7303a1ddf 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/conditions.py b/tests/integration/conditions.py index b25010ca2..80e3e5855 100644 --- a/tests/integration/conditions.py +++ b/tests/integration/conditions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 58460e18f..c0b699055 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/test_definitions.py b/tests/integration/test_definitions.py index f59acf237..273fa7c56 100644 --- a/tests/integration/test_definitions.py +++ b/tests/integration/test_definitions.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.11.0. +# This file is part of Merlin, Version: 1.11.1. # # For details, see https://github.com/LLNL/merlin. # From 5dc82061f235d1710518158b20819bc51ef022ce Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Thu, 2 Nov 2023 11:45:48 -0700 Subject: [PATCH 22/29] Add Pytest Fixtures to Test Suite (#456) * begin work on integration refactor; create fixtures and initial tests * update CHANGELOG and run fix-style * add pytest fixtures and README explaining them * add tests to demonstrate how to use the fixtures * move/rename some files and modify integration's README * add password change to redis.pass file * fix lint issues * modify redis pwd for test server to be constant for each test * fix lint issue only caught on github ci --- CHANGELOG.md | 5 + merlin/common/tasks.py | 2 +- merlin/server/server_commands.py | 3 + merlin/study/celeryadapter.py | 34 +- requirements/dev.txt | 1 + tests/README.md | 152 +++++++++ tests/conftest.py | 301 ++++++++++++++++++ .../{test_definitions.py => definitions.py} | 0 tests/integration/run_tests.py | 6 +- tests/unit/study/test_celeryadapter.py | 160 ++++++++++ 10 files changed, 647 insertions(+), 17 deletions(-) create mode 100644 tests/README.md create mode 100644 tests/conftest.py rename tests/integration/{test_definitions.py => definitions.py} (100%) create mode 100644 tests/unit/study/test_celeryadapter.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d0ef2ae6..9ca916ea9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] +### Added +- Pytest fixtures in the `conftest.py` file of the integration test suite +- Tests for the `celeryadapter.py` module + ## [1.11.1] ### Fixed - Typo in `batch.py` that caused lsf launches to fail (`ALL_SGPUS` changed to `ALL_GPUS`) diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index 2bd77d2ad..8292e559f 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -480,7 +480,7 @@ def expand_tasks_with_samples( # pylint: disable=R0913,R0914 if not found_tasks: for next_index_path, next_index in sample_index.traverse(conditional=condition): LOG.info( - f"generating next step for range {next_index.min}:{next_index.max} {next_index.max-next_index.min}" + f"generating next step for range {next_index.min}:{next_index.max} {next_index.max - next_index.min}" ) next_index.name = next_index_path diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index a28776577..c244c9eca 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -92,6 +92,9 @@ def config_server(args: Namespace) -> None: # pylint: disable=R0912 redis_users = RedisUsers(server_config.container.get_user_file_path()) redis_users.set_password("default", args.password) redis_users.write() + pass_file = server_config.container.get_pass_file_path() + with open(pass_file, "w") as pfile: + pfile.write(args.password) redis_config.set_directory(args.directory) diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index a6707d952..cd9714dff 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -37,6 +37,7 @@ import subprocess import time from contextlib import suppress +from typing import Dict, List, Optional from merlin.study.batch import batch_check_parallel, batch_worker_launch from merlin.utils import apply_list_of_regex, check_machines, get_procs, get_yaml_var, is_running @@ -69,23 +70,31 @@ def run_celery(study, run_mode=None): queue_merlin_study(study, adapter_config) -def get_running_queues(): +def get_running_queues(celery_app_name: str, test_mode: bool = False) -> List[str]: """ - Check for running celery workers with -Q queues - and return a unique list of the queues + Check for running celery workers by looking at the currently running processes. + If there are running celery workers, we'll pull the queues from the -Q tag in the + process command. The list returned here will contain only unique celery queue names. + This must be run on the allocation where the workers are running. - Must be run on the allocation where the workers are running + :param `celery_app_name`: The name of the celery app (typically merlin here unless testing) + :param `test_mode`: If True, run this function in test mode + :returns: A unique list of celery queues with workers attached to them """ running_queues = [] - if not is_running("celery worker"): + if not is_running(f"{celery_app_name} worker"): return running_queues - procs = get_procs("celery") + proc_name = "celery" if not test_mode else "sh" + procs = get_procs(proc_name) for _, lcmd in procs: lcmd = list(filter(None, lcmd)) cmdline = " ".join(lcmd) if "-Q" in cmdline: + if test_mode: + echo_cmd = lcmd.pop(2) + lcmd.extend(echo_cmd.split()) running_queues.extend(lcmd[lcmd.index("-Q") + 1].split(",")) running_queues = list(set(running_queues)) @@ -155,19 +164,20 @@ def get_active_workers(app): return worker_queue_map -def celerize_queues(queues): +def celerize_queues(queues: List[str], config: Optional[Dict] = None): """ Celery requires a queue tag to be prepended to their queues so this function will 'celerize' every queue in a list you provide it by prepending the queue tag. - :param `queues`: A list of queues that need the queue - tag prepended. + :param `queues`: A list of queues that need the queue tag prepended. + :param `config`: A dict of configuration settings """ - from merlin.config.configfile import CONFIG # pylint: disable=C0415 + if config is None: + from merlin.config.configfile import CONFIG as config # pylint: disable=C0415 for i, queue in enumerate(queues): - queues[i] = f"{CONFIG.celery.queue_tag}{queue}" + queues[i] = f"{config.celery.queue_tag}{queue}" def _build_output_table(worker_list, output_table): @@ -462,7 +472,7 @@ def start_celery_workers(spec, steps, celery_args, disable_logs, just_return_com running_queues.extend(local_queues) queues = queues.split(",") if not overlap: - running_queues.extend(get_running_queues()) + running_queues.extend(get_running_queues("merlin")) # Cache the queues from this worker to use to test # for existing queues in any subsequent workers. # If overlap is True, then do not check the local queues. diff --git a/requirements/dev.txt b/requirements/dev.txt index 9321694f8..895a89249 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -10,3 +10,4 @@ twine sphinx>=2.0.0 alabaster johnnydep +deepdiff diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 000000000..a6bf7005a --- /dev/null +++ b/tests/README.md @@ -0,0 +1,152 @@ +# Tests + +This directory utilizes pytest to create and run our test suite. +Here we use pytest fixtures to create a local redis server and a celery app for testing. + +This directory is organized like so: +- `conftest.py` - The script containing all fixtures for our tests +- `unit/` - The directory containing unit tests + - `test_*.py` - The actual test scripts to run +- `integration/` - The directory containing integration tests + + - `definitions.py` - The test definitions + - `run_tests.py` - The script to run the tests defined in `definitions.py` + - `conditions.py` - The conditions to test against + +## How to Run + +Before running any tests: + +1. Activate your virtual environment with Merlin's dev requirements installed +2. Navigate to the tests folder where this README is located + +To run the entire test suite: + +``` +python -m pytest +``` + +To run a specific test file: + +``` +python -m pytest /path/to/test_specific_file.py +``` + +To run a certain test class within a specific test file: + +``` +python -m pytest /path/to/test_specific_file.py::TestCertainClass +``` + +To run one unique test: + +``` +python -m pytest /path/to/test_specific_file.py::TestCertainClass::test_unique_test +``` + +## Killing the Test Server + +In case of an issue with the test suite, or if you stop the tests with `ctrl+C`, you may need to stop +the server manually. This can be done with: + +``` +redis-cli +127.0.0.1:6379> AUTH merlin-test-server +127.0.0.1:6379> shutdown +not connected> quit +``` + +## The Fixture Process Explained + +Pytest fixtures play a fundamental role in establishing a consistent foundation for test execution, +thus ensuring reliable and predictable test outcomes. This section will delve into essential aspects +of these fixtures, including how to integrate fixtures into tests, the utilization of fixtures within other fixtures, +their scope, and the yielding of fixture results. + +### How to Integrate Fixtures Into Tests + +Probably the most important part of fixtures is understanding how to use them. Luckily, this process is very +simple and can be dumbed down to 2 steps: + +1. Create a fixture in the `conftest.py` file by using the `@pytest.fixture` decorator. For example: + +``` +@pytest.fixture +def dummy_fixture(): + return "hello world" +``` + +2. Use it as an argument in a test function (you don't even need to import it!): + +``` +def test_dummy(dummy_fixture): + assert dummy_fixture == "hello world" +``` + +For more information, see [Pytest's documentation](https://docs.pytest.org/en/7.1.x/how-to/fixtures.html#how-to-use-fixtures). + +### Fixtureception + +One of the coolest and most useful aspects of fixtures that we utilize in this test suite is the ability for +fixtures to be used within other fixtures. For more info on this from pytest, see +[here](https://docs.pytest.org/en/7.1.x/how-to/fixtures.html#fixtures-can-request-other-fixtures). + +Pytest will handle fixtures within fixtures in a stack-based way. Let's look at how creating the `redis_pass` +fixture from our `conftest.py` file works in order to illustrate the process. +1. First, we start by telling pytest that we want to use the `redis_pass` fixture by providing it as an argument +to a test/fixture: + +``` +def test_example(redis_pass): + ... +``` + +2. Now pytest will find the `redis_pass` fixture and put it at the top of the stack to be created. However, +it'll see that this fixture requires another fixture `merlin_server_dir` as an argument: + +``` +@pytest.fixture(scope="session") +def redis_pass(merlin_server_dir): + ... +``` + +3. Pytest then puts the `merlin_server_dir` fixture at the top of the stack, but similarly it sees that this fixture +requires yet another fixture `temp_output_dir`: + +``` +@pytest.fixture(scope="session") +def merlin_server_dir(temp_output_dir: str) -> str: + ... +``` + +4. This process continues until it reaches a fixture that doesn't require any more fixtures. At this point the base +fixture is created and pytest will start working its way back up the stack to the first fixture it looked at (in this +case `redis_pass`). + +5. Once all required fixtures are created, execution will be returned to the test which can now access the fixture +that was requested (`redis_pass`). + +As you can see, if we have to re-do this process for every test it could get pretty time intensive. This is where fixture +scopes come to save the day. + +### Fixture Scopes + +There are several different scopes that you can set for fixtures. The majority of our fixtures use a `session` +scope so that we only have to create the fixtures one time (as some of them can take a few seconds to set up). +The goal is to create fixtures with the most general use-case in mind so that we can re-use them for larger +scopes, which helps with efficiency. + +For more info on scopes, see +[Pytest's Fixture Scope documentation](https://docs.pytest.org/en/6.2.x/fixture.html#scope-sharing-fixtures-across-classes-modules-packages-or-session). + +### Yielding Fixtures + +In several fixtures throughout our test suite, we need to run some sort of teardown for the fixture. For example, +once we no longer need the `redis_server` fixture, we need to shut the server down so it stops using resources. +This is where yielding fixtures becomes extremely useful. + +Using the `yield` keyword allows execution to be returned to a test that needs the fixture once the feature has +been set up. After all tests using the fixture have been ran, execution will return to the fixture for us to run +our teardown code. + +For more information on yielding fixtures, see [Pytest's documentation](https://docs.pytest.org/en/7.1.x/how-to/fixtures.html#teardown-cleanup-aka-fixture-finalization). \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..a496175eb --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,301 @@ +############################################################################### +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.11.1. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### +""" +This module contains pytest fixtures to be used throughout the entire +integration test suite. +""" +import multiprocessing +import os +import subprocess +from time import sleep +from typing import Dict, List + +import pytest +import redis +from _pytest.tmpdir import TempPathFactory +from celery import Celery + + +class RedisServerError(Exception): + """ + Exception to signal that the server wasn't pinged properly. + """ + + +class ServerInitError(Exception): + """ + Exception to signal that there was an error initializing the server. + """ + + +@pytest.fixture(scope="session") +def temp_output_dir(tmp_path_factory: TempPathFactory) -> str: + """ + This fixture will create a temporary directory to store output files of integration tests. + The temporary directory will be stored at /tmp/`whoami`/pytest-of-`whoami`/. There can be at most + 3 temp directories in this location so upon the 4th test run, the 1st temp directory will be removed. + + :param tmp_path_factory: A built in factory with pytest to help create temp paths for testing + :yields: The path to the temp output directory we'll use for this test run + """ + # Log the cwd, then create and move into the temporary one + cwd = os.getcwd() + temp_integration_outfile_dir = tmp_path_factory.mktemp("integration_outfiles_") + os.chdir(temp_integration_outfile_dir) + + yield temp_integration_outfile_dir + + # Move back to the directory we started at + os.chdir(cwd) + + +@pytest.fixture(scope="session") +def redis_pass() -> str: + """ + This fixture represents the password to the merlin test server. + + :returns: The redis password for our test server + """ + return "merlin-test-server" + + +@pytest.fixture(scope="session") +def merlin_server_dir(temp_output_dir: str, redis_pass: str) -> str: # pylint: disable=redefined-outer-name + """ + This fixture will initialize the merlin server (i.e. create all the files we'll + need to start up a local redis server). It will return the path to the directory + containing the files needed for the server to start up. + + :param temp_output_dir: The path to the temporary output directory we'll be using for this test run + :param redis_pass: The password to the test redis server that we'll create here + :returns: The path to the merlin_server directory with the server configurations + """ + # Initialize the setup for the local redis server + # We'll also set the password to 'merlin-test-server' so it'll be easy to shutdown if there's an issue + subprocess.run(f"merlin server init; merlin server config -pwd {redis_pass}", shell=True, capture_output=True, text=True) + + # Check that the merlin server was initialized properly + server_dir = f"{temp_output_dir}/merlin_server" + if not os.path.exists(server_dir): + raise ServerInitError("The merlin server was not initialized properly.") + + return server_dir + + +@pytest.fixture(scope="session") +def redis_server(merlin_server_dir: str, redis_pass: str) -> str: # pylint: disable=redefined-outer-name,unused-argument + """ + Start a redis server instance that runs on localhost:6379. This will yield the + redis server uri that can be used to create a connection with celery. + + :param merlin_server_dir: The directory to the merlin test server configuration. + This will not be used here but we need the server configurations before we can + start the server. + :param redis_pass: The raw redis password stored in the redis.pass file + :yields: The local redis server uri + """ + # Start the local redis server + try: + subprocess.run("merlin server start", shell=True, capture_output=True, text=True, timeout=5) + except subprocess.TimeoutExpired: + pass + + # Ensure the server started properly + host = "localhost" + port = 6379 + database = 0 + username = "default" + redis_client = redis.Redis(host=host, port=port, db=database, password=redis_pass, username=username) + if not redis_client.ping(): + raise RedisServerError("The redis server could not be pinged. Check that the server is running with 'ps ux'.") + + # Hand over the redis server url to any other fixtures/tests that need it + redis_server_uri = f"redis://{username}:{redis_pass}@{host}:{port}/{database}" + yield redis_server_uri + + # Kill the server; don't run this until all tests are done (accomplished with 'yield' above) + kill_process = subprocess.run("merlin server stop", shell=True, capture_output=True, text=True) + assert "Merlin server terminated." in kill_process.stderr + + +@pytest.fixture(scope="session") +def celery_app(redis_server: str) -> Celery: # pylint: disable=redefined-outer-name + """ + Create the celery app to be used throughout our integration tests. + + :param redis_server: The redis server uri we'll use to connect to redis + :returns: The celery app object we'll use for testing + """ + return Celery("test_app", broker=redis_server, backend=redis_server) + + +@pytest.fixture(scope="session") +def worker_queue_map() -> Dict[str, str]: + """ + Worker and queue names to be used throughout tests + + :returns: A dict of dummy worker/queue associations + """ + return {f"test_worker_{i}": f"test_queue_{i}" for i in range(3)} + + +def are_workers_ready(app: Celery, num_workers: int, verbose: bool = False) -> bool: + """ + Check to see if the workers are up and running yet. + + :param app: The celery app fixture that's connected to our redis server + :param num_workers: An int representing the number of workers we're looking to have started + :param verbose: If true, enable print statements to show where we're at in execution + :returns: True if all workers are running. False otherwise. + """ + app_stats = app.control.inspect().stats() + if verbose: + print(f"app_stats: {app_stats}") + return app_stats is not None and len(app_stats) == num_workers + + +def wait_for_worker_launch(app: Celery, num_workers: int, verbose: bool = False): + """ + Poll the workers over a fixed interval of time. If the workers don't show up + within the time limit then we'll raise a timeout error. Otherwise, the workers + are up and running and we can continue with our tests. + + :param app: The celery app fixture that's connected to our redis server + :param num_workers: An int representing the number of workers we're looking to have started + :param verbose: If true, enable print statements to show where we're at in execution + """ + max_wait_time = 2 # Maximum wait time in seconds + wait_interval = 0.5 # Interval between checks in seconds + waited_time = 0 + + if verbose: + print("waiting for workers to launch...") + + # Wait until all workers are ready + while not are_workers_ready(app, num_workers, verbose=verbose) and waited_time < max_wait_time: + sleep(wait_interval) + waited_time += wait_interval + + # If all workers are not ready after the maximum wait time, raise an error + if not are_workers_ready(app, num_workers, verbose=verbose): + raise TimeoutError("Celery workers did not start within the expected time.") + + if verbose: + print("workers launched") + + +def shutdown_processes(worker_processes: List[multiprocessing.Process], echo_processes: List[subprocess.Popen]): + """ + Given lists of processes, shut them all down. Worker processes were created with the + multiprocessing library and echo processes were created with the subprocess library, + so we have to shut them down slightly differently. + + :param worker_processes: A list of worker processes to terminate + :param echo_processes: A list of echo processes to terminate + """ + # Worker processes were created with the multiprocessing library + for worker_process in worker_processes: + # Try to terminate the process gracefully + worker_process.terminate() + process_exit_code = worker_process.join(timeout=3) + + # If it won't terminate then force kill it + if process_exit_code is None: + worker_process.kill() + + # Gracefully terminate the echo processes + for echo_process in echo_processes: + echo_process.terminate() + echo_process.wait() + + # The echo processes will spawn 3 sleep inf processes that we also need to kill + subprocess.run("ps ux | grep 'sleep inf' | grep -v grep | awk '{print $2}' | xargs kill", shell=True) + + +def start_worker(app: Celery, worker_launch_cmd: List[str]): + """ + This is where a worker is actually started. Each worker maintains control of a process until + we tell it to stop, that's why we have to use the multiprocessing library for this. We have to use + app.worker_main instead of the normal "celery -A worker" command to launch the workers + since our celery app is created in a pytest fixture and is unrecognizable by the celery command. + For each worker, the output of it's logs are sent to + /tmp/`whoami`/pytest-of-`whoami`/pytest-current/integration_outfiles_current/ under a file with a name + similar to: test_worker_*.log. + NOTE: pytest-current/ will have the results of the most recent test run. If you want to see a previous run + check under pytest-/. HOWEVER, only the 3 most recent test runs will be saved. + + :param app: The celery app fixture that's connected to our redis server + :param worker_launch_cmd: The command to launch a worker + """ + app.worker_main(worker_launch_cmd) + + +@pytest.fixture(scope="class") +def launch_workers(celery_app: Celery, worker_queue_map: Dict[str, str]): # pylint: disable=redefined-outer-name + """ + Launch the workers on the celery app fixture using the worker and queue names + defined in the worker_queue_map fixture. + + :param celery_app: The celery app fixture that's connected to our redis server + :param worker_queue_map: A dict where the keys are worker names and the values are queue names + """ + # Create the processes that will start the workers and store them in a list + worker_processes = [] + echo_processes = [] + for worker, queue in worker_queue_map.items(): + worker_launch_cmd = ["worker", "-n", worker, "-Q", queue, "--concurrency", "1", f"--logfile={worker}.log"] + + # We have to use this dummy echo command to simulate a celery worker command that will show up with 'ps ux' + # We'll sleep for infinity here and then kill this process during shutdown + echo_process = subprocess.Popen( # pylint: disable=consider-using-with + f"echo 'celery test_app {' '.join(worker_launch_cmd)}'; sleep inf", shell=True + ) + echo_processes.append(echo_process) + + # We launch workers in their own process since they maintain control of a process until we stop them + worker_process = multiprocessing.Process(target=start_worker, args=(celery_app, worker_launch_cmd)) + worker_process.start() + worker_processes.append(worker_process) + + # Ensure that the workers start properly before letting tests use them + try: + num_workers = len(worker_queue_map) + wait_for_worker_launch(celery_app, num_workers, verbose=False) + except TimeoutError as exc: + # If workers don't launch in time, we need to make sure these processes stop + shutdown_processes(worker_processes, echo_processes) + raise exc + + # Give control to the tests that need to use workers + yield + + # Shut down the workers and terminate the processes + celery_app.control.broadcast("shutdown", destination=list(worker_queue_map.keys())) + shutdown_processes(worker_processes, echo_processes) diff --git a/tests/integration/test_definitions.py b/tests/integration/definitions.py similarity index 100% rename from tests/integration/test_definitions.py rename to tests/integration/definitions.py diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index c0b699055..fcdb9e0b2 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -39,10 +39,8 @@ from contextlib import suppress from subprocess import TimeoutExpired, run -# Pylint complains that we didn't install this module but it's defined locally so ignore -from test_definitions import OUTPUT_DIR, define_tests # pylint: disable=E0401 - from merlin.display import tabulate_info +from tests.integration.definitions import OUTPUT_DIR, define_tests # pylint: disable=E0401 def get_definition_issues(test): @@ -237,7 +235,7 @@ def run_tests(args, tests): # pylint: disable=R0914 total += 1 continue dot_length = 50 - len(test_name) - len(str(test_label)) - print(f"TEST {test_label}: {test_name}{'.'*dot_length}", end="") + print(f"TEST {test_label}: {test_name}{'.' * dot_length}", end="") # Check the format of the test definition definition_issues = get_definition_issues(test) if definition_issues: diff --git a/tests/unit/study/test_celeryadapter.py b/tests/unit/study/test_celeryadapter.py new file mode 100644 index 000000000..82e8401e6 --- /dev/null +++ b/tests/unit/study/test_celeryadapter.py @@ -0,0 +1,160 @@ +############################################################################### +# Copyright (c) 2023, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory +# Written by the Merlin dev team, listed in the CONTRIBUTORS file. +# +# +# LLNL-CODE-797170 +# All rights reserved. +# This file is part of Merlin, Version: 1.11.1. +# +# For details, see https://github.com/LLNL/merlin. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +############################################################################### +""" +Tests for the celeryadapter module. +""" +from typing import Dict + +import celery + +from merlin.config import Config +from merlin.study import celeryadapter + + +class TestActiveQueues: + """ + This class will test queue related functions in the celeryadapter.py module. + It will run tests where we need active queues to interact with. + """ + + def test_query_celery_queues(self, launch_workers: "Fixture"): # noqa: F821 + """ + Test the query_celery_queues function by providing it with a list of active queues. + This should return a list of tuples. Each tuple will contain information + (name, num jobs, num consumers) for each queue that we provided. + """ + # TODO Modify query_celery_queues so the output for a redis broker is the same + # as the output for rabbit broker + + def test_get_running_queues(self, launch_workers: "Fixture", worker_queue_map: Dict[str, str]): # noqa: F821 + """ + Test the get_running_queues function with queues active. + This should return a list of active queues. + + :param `launch_workers`: A pytest fixture that launches celery workers for us to interact with + :param `worker_queue_map`: A pytest fixture that returns a dict of workers and queues + """ + result = celeryadapter.get_running_queues("test_app", test_mode=True) + assert sorted(result) == sorted(list(worker_queue_map.values())) + + def test_get_queues_active( + self, celery_app: celery.Celery, launch_workers: "Fixture", worker_queue_map: Dict[str, str] # noqa: F821 + ): + """ + Test the get_queues function with queues active. + This should return a tuple where the first entry is a dict of queue info + and the second entry is a list of worker names. + + :param `celery_app`: A pytest fixture for the test Celery app + :param `launch_workers`: A pytest fixture that launches celery workers for us to interact with + :param `worker_queue_map`: A pytest fixture that returns a dict of workers and queues + """ + # Start the queues and run the test + queue_result, worker_result = celeryadapter.get_queues(celery_app) + + # Ensure we got output before looping + assert len(queue_result) == len(worker_result) == 3 + + for worker, queue in worker_queue_map.items(): + # Check that the entry in the queue_result dict for this queue is correct + assert queue in queue_result + assert len(queue_result[queue]) == 1 + assert worker in queue_result[queue][0] + + # Remove this entry from the queue_result dict + del queue_result[queue] + + # Check that this worker was added to the worker_result list + worker_found = False + for worker_name in worker_result[:]: + if worker in worker_name: + worker_found = True + worker_result.remove(worker_name) + break + assert worker_found + + # Ensure there was no extra output that we weren't expecting + assert queue_result == {} + assert worker_result == [] + + +class TestInactiveQueues: + """ + This class will test queue related functions in the celeryadapter.py module. + It will run tests where we don't need any active queues to interact with. + """ + + def test_query_celery_queues(self): + """ + Test the query_celery_queues function by providing it with a list of inactive queues. + This should return a list of strings. Each string will give a message saying that a + particular queue was inactive + """ + # TODO Modify query_celery_queues so the output for a redis broker is the same + # as the output for rabbit broker + + def test_celerize_queues(self, worker_queue_map: Dict[str, str]): + """ + Test the celerize_queues function. This should add the celery queue_tag + to the front of the queues we provide it. + + :param `worker_queue_map`: A pytest fixture that returns a dict of workers and queues + """ + # Create variables to be used in the test + queue_tag = "[merlin]_" + queues_to_check = list(worker_queue_map.values()) + dummy_config = Config({"celery": {"queue_tag": queue_tag}}) + + # Run the test + celeryadapter.celerize_queues(queues_to_check, dummy_config) + + # Ensure the queue tag was added to every queue + for queue in queues_to_check: + assert queue_tag in queue + + def test_get_running_queues(self): + """ + Test the get_running_queues function with no queues active. + This should return an empty list. + """ + result = celeryadapter.get_running_queues("test_app", test_mode=True) + assert result == [] + + def test_get_queues(self, celery_app: celery.Celery): + """ + Test the get_queues function with no queues active. + This should return a tuple where the first entry is an empty dict + and the second entry is an empty list. + + :param `celery_app`: A pytest fixture for the test Celery app + """ + queue_result, worker_result = celeryadapter.get_queues(celery_app) + assert queue_result == {} + assert worker_result == [] From 38651f2650e8aba97552c4575e97d66be3205545 Mon Sep 17 00:00:00 2001 From: Brian Gunnarson <49216024+bgunnar5@users.noreply.github.com> Date: Tue, 14 Nov 2023 17:38:13 -0800 Subject: [PATCH 23/29] Bugfix for WEAVE CI (#457) * begin work on integration refactor; create fixtures and initial tests * update CHANGELOG and run fix-style * add pytest fixtures and README explaining them * add tests to demonstrate how to use the fixtures * move/rename some files and modify integration's README * add password change to redis.pass file * fix lint issues * modify redis pwd for test server to be constant for each test * fix lint issue only caught on github ci * add fix for merlin server startup * update CHANGELOG --- CHANGELOG.md | 1 + tests/conftest.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ca916ea9..9db50369a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added - Pytest fixtures in the `conftest.py` file of the integration test suite + - NOTE: an export command `export LC_ALL='C'` had to be added to fix a bug in the WEAVE CI. This can be removed when we resolve this issue for the `merlin server` command - Tests for the `celeryadapter.py` module ## [1.11.1] diff --git a/tests/conftest.py b/tests/conftest.py index a496175eb..88932c5db 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -123,7 +123,8 @@ def redis_server(merlin_server_dir: str, redis_pass: str) -> str: # pylint: dis """ # Start the local redis server try: - subprocess.run("merlin server start", shell=True, capture_output=True, text=True, timeout=5) + # Need to set LC_ALL='C' before starting the server or else redis causes a failure + subprocess.run("export LC_ALL='C'; merlin server start", shell=True, capture_output=True, text=True, timeout=5) except subprocess.TimeoutExpired: pass From 83f4a7c15a1f4ca30e8a1ee1e4ac8ad5f743a5c7 Mon Sep 17 00:00:00 2001 From: Dylan Thomas Cliche Date: Thu, 30 Nov 2023 16:12:08 -0800 Subject: [PATCH 24/29] A new example for having the workflow stop the workers itself. This helps to alleviate the need to constantly check an ensemble to stop the workers when the workflow finishes. See README file for further details. --- .../examples/workflows/stop_ensemble/README | 72 +++++++++++++++++++ .../stop_ensemble/hello_samples.yaml | 51 +++++++++++++ .../workflows/stop_ensemble/make_samples.py | 22 ++++++ .../workflows/stop_ensemble/quitScript.sh | 31 ++++++++ .../workflows/stop_ensemble/requirements.txt | 2 + .../workflows/stop_ensemble/workers.sbatch | 72 +++++++++++++++++++ 6 files changed, 250 insertions(+) create mode 100644 merlin/examples/workflows/stop_ensemble/README create mode 100644 merlin/examples/workflows/stop_ensemble/hello_samples.yaml create mode 100644 merlin/examples/workflows/stop_ensemble/make_samples.py create mode 100644 merlin/examples/workflows/stop_ensemble/quitScript.sh create mode 100644 merlin/examples/workflows/stop_ensemble/requirements.txt create mode 100644 merlin/examples/workflows/stop_ensemble/workers.sbatch diff --git a/merlin/examples/workflows/stop_ensemble/README b/merlin/examples/workflows/stop_ensemble/README new file mode 100644 index 000000000..d5e3c4891 --- /dev/null +++ b/merlin/examples/workflows/stop_ensemble/README @@ -0,0 +1,72 @@ ++----------------------------------------------------+ +| | +| Stop Workers Example | +| | +| Description: Hello World example including a quit | +| script to stop running and dependent | +| jobs running the Merlin workflow. | +| | +| Important Scripts: -workers.sbatch | +| -hello_samples.yaml | +| -quitScript.sh | +| | +| Created By: Dylan Cliche (cliche1@llnl.gov) | +| Updated On: 08/14/2023 | ++----------------------------------------------------+ + + +========================================================== + Running the Example +========================================================== + +1) merlin run +2) sbatch workers.sbatch + +Notes: - is your yaml file including + the extension ".yaml". + -The second input parameter to workers.sbatch tells + slurm to either run a diasy chain of dependency + jobs ("true") or no dependency jobs ("false" or + no second input parameter at all). + + +========================================================== + What is Going On +========================================================== + +When the job is allocated, the typical hello_samples will +execute via step_1 and step_2 within the yaml file. After, +a stopWorkers step is executed that will copy the +quitScript.sh bash file from $(SPECROOT) and obtain the +specfile and the specRoot. + +These two variables are passed into quitSript.sh +which will find all job IDs of both running and awaiting +job allocations, loop through them, and cancel every +single one that deals with the +ran from $(SPECROOT) defined in steps 1 and 2 in the +"Running the Example" section above. + + +========================================================== + Important Notes +========================================================== + +1) quitScript.sh is submitted via sbatch instead of a + straight source or bash -c call because otherwise the + script will stop the workers immediately and the YAML + file then stops, but the task would never obtain a + successfull exit. Therefore, it would have to be purged + each time before starting a new Merlin run. By + submitting it via sbatch, once it is submitted the task + can exit successfully before the workers are stopped. + +2) Make sure the stopWorkers task is the very last task + in your workflow as it will stop any and all jobs + that have been allocated for the workflow. + +3) The virtual environment name and path are defined + in workers.sbatch. Change these to your specific needs + prior to running. + + diff --git a/merlin/examples/workflows/stop_ensemble/hello_samples.yaml b/merlin/examples/workflows/stop_ensemble/hello_samples.yaml new file mode 100644 index 000000000..e801df775 --- /dev/null +++ b/merlin/examples/workflows/stop_ensemble/hello_samples.yaml @@ -0,0 +1,51 @@ +description: + name: hello_samples + description: An example to stop ensembles automatically using hello world + +env: + variables: + N_SAMPLES: 10 + +global.parameters: + GREET: + values : ["hello","hola"] + label : GREET.%% + +study: + - name: step_1 + description: say hello + run: + cmd: echo "$(GREET), $(WORLD)!" + + - name: step_2 + description: print a success message + run: + cmd: print("Hurrah, we did it!") + depends: [step_1_*] + shell: /usr/bin/env python3 + + - name: stopEnsemble + description: Stop any and all workers attached to this specification file + run: + cmd: | + cp $(SPECROOT)/quitScript.sh ./ + + specFile=$(basename $(MERLIN_SPEC_ORIGINAL_TEMPLATE)) + specFile=${specFile/.orig.yaml/.yaml} + + echo $(SPECROOT) + echo $specFile + + sbatch ./quitScript.sh $(SPECROOT) $specFile + + #Give successful exit since batch job was canceled + #and can't do it + exit $(MERLIN_SUCCESS) + depends: [step_2] + +merlin: + samples: + generate: + cmd: python3 $(SPECROOT)/make_samples.py --filepath=$(MERLIN_INFO)/samples.csv --number=$(N_SAMPLES) + file: $(MERLIN_INFO)/samples.csv + column_labels: [WORLD] diff --git a/merlin/examples/workflows/stop_ensemble/make_samples.py b/merlin/examples/workflows/stop_ensemble/make_samples.py new file mode 100644 index 000000000..3b7b5b398 --- /dev/null +++ b/merlin/examples/workflows/stop_ensemble/make_samples.py @@ -0,0 +1,22 @@ +import argparse + +import names +import numpy as np + + +# argument parsing +parser = argparse.ArgumentParser(description="Make some samples (names of people).") +parser.add_argument("--number", type=int, action="store", help="the number of samples you want to make") +parser.add_argument("--filepath", type=str, help="output file") +args = parser.parse_args() + +# sample making +all_names = np.loadtxt(names.FILES["first:female"], dtype=str, usecols=0) +selected_names = np.random.choice(all_names, size=args.number) + +result = "" +name_list = list(selected_names) +result = "\n".join(name_list) + +with open(args.filepath, "w") as f: + f.write(result) diff --git a/merlin/examples/workflows/stop_ensemble/quitScript.sh b/merlin/examples/workflows/stop_ensemble/quitScript.sh new file mode 100644 index 000000000..030650ac9 --- /dev/null +++ b/merlin/examples/workflows/stop_ensemble/quitScript.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +#SBATCH -N 1 +#SBATCH --ntasks-per-node=1 +#SBATCH -J stopWorkers +#SBATCH -t 00:00:05 +#SBATCH -o merlin_StopWorker_%j.out + +# Turn off core files to work around flux exec issue. +ulimit -c 0 + + +specRoot=$1 +targetSpecFile=$2 + +for JOB2CHECK in $(squeue --me --format="%F" | grep -v "ARRAY") +do + if [[ $JOB2CHECK != $SLURM_JOB_ID ]] + then + if [[ ! -z $(scontrol show job $JOB2CHECK | \ + grep "Command=$specRoot" | \ + grep -w $targetSpecFile) ]] + then + cancelJOB_NAME=$(squeue --me --job=$JOB2CHECK --format=%j | sed -n '2p') + echo $cancelJOB_NAME + echo "Job to Cancel::::Job Name: $cancelJOB_NAME Job ID: $JOB2CHECK" + echo "canceling job" + scancel $JOB2CHECK + fi + fi +done diff --git a/merlin/examples/workflows/stop_ensemble/requirements.txt b/merlin/examples/workflows/stop_ensemble/requirements.txt new file mode 100644 index 000000000..684db4dc8 --- /dev/null +++ b/merlin/examples/workflows/stop_ensemble/requirements.txt @@ -0,0 +1,2 @@ +names +numpy diff --git a/merlin/examples/workflows/stop_ensemble/workers.sbatch b/merlin/examples/workflows/stop_ensemble/workers.sbatch new file mode 100644 index 000000000..a79038e7b --- /dev/null +++ b/merlin/examples/workflows/stop_ensemble/workers.sbatch @@ -0,0 +1,72 @@ +#!/bin/bash + +#SBATCH -N 1 +#SBATCH --ntasks-per-node=36 +#SBATCH -J stopEnsembleExample +#SBATCH -t 00:10:00 +#SBATCH -o merlin_workers_%j.out +#SBATCH -p pdebug + +# Turn off core files to work around flux exec issue. +ulimit -c 0 + +doDaisyChain=false +YAML=default + +# Get this filename +WORKER_SCRIPT=$(scontrol show job $SLURM_JOB_ID | grep -hi "Command=" | xargs -0 basename) + +if [[ $# -gt 0 ]] +then + YAML=$1 + + if [[ $# -gt 1 ]] + then + if [[ $2 == 1 || $2 == true || $2 == True || $2 == yes || $2 == Yes ]] + then + doDaisyChain=true + fi + fi +fi + + +echo "Spec Filename: $YAML" + +MERLIN_PATH=/usr/workspace/cliche1/MERLIN +VENV_NAME=merlin_venv_toss_3_x86_64_ib_py_3_8 + +# Activate the virtual environment +source ${MERLIN_PATH}/${VENV_NAME}/bin/activate + +# Show the workers command +merlin run-workers ${YAML} --echo + +# Start workers to run the tasks in the broker +merlin run-workers ${YAML} + +# Check that the YAML file was correct. Otherwise cancel job +outFile=$(scontrol show job $SLURM_JOB_ID | grep -i 'stdout=') +outFile=${outFile#*=} +isBadYAML=$(head -n 1 $outFile | grep -i "is not a valid filepath" | wc -l) +if [[ isBadYAML -eq 1 ]] +then + echo "Incorrect YAML File: $YAML" + echo "Canceling Job" + scancel $SLURM_JOB_ID +fi + + +# Start dependency job if this one times out +# The new job name will contain the job id of the job that +# created the depenendency +if [[ $doDaisyChain = true ]] +then + JOB_NAME_BASE=${SLURM_JOB_NAME%%-d-*} + JOB_NAME="$JOB_NAME_BASE-d-${SLURM_JOB_ID}" + sbatch -J $JOB_NAME --depend=afterany:${SLURM_JOB_ID} ${WORKER_SCRIPT} $YAML $doDaisyChain +fi + +# Keep the allocation alive +sleep inf + + From adf5d661a0ec280884dbab28749142ce5551a28e Mon Sep 17 00:00:00 2001 From: Dylan Thomas Cliche Date: Mon, 4 Dec 2023 09:31:12 -0800 Subject: [PATCH 25/29] Updated --- merlin/examples/workflows/stop_ensemble/README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/merlin/examples/workflows/stop_ensemble/README b/merlin/examples/workflows/stop_ensemble/README index d5e3c4891..6a3eadd9c 100644 --- a/merlin/examples/workflows/stop_ensemble/README +++ b/merlin/examples/workflows/stop_ensemble/README @@ -1,6 +1,6 @@ +----------------------------------------------------+ | | -| Stop Workers Example | +| Stop Ensemble Example | | | | Description: Hello World example including a quit | | script to stop running and dependent | From d3ee3d58dee92a7946fa976b81e909b06f05626a Mon Sep 17 00:00:00 2001 From: Dylan Thomas Cliche Date: Mon, 4 Dec 2023 10:36:40 -0800 Subject: [PATCH 26/29] Made general --- merlin/examples/workflows/stop_ensemble/workers.sbatch | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/merlin/examples/workflows/stop_ensemble/workers.sbatch b/merlin/examples/workflows/stop_ensemble/workers.sbatch index a79038e7b..7ceb31994 100644 --- a/merlin/examples/workflows/stop_ensemble/workers.sbatch +++ b/merlin/examples/workflows/stop_ensemble/workers.sbatch @@ -32,8 +32,8 @@ fi echo "Spec Filename: $YAML" -MERLIN_PATH=/usr/workspace/cliche1/MERLIN -VENV_NAME=merlin_venv_toss_3_x86_64_ib_py_3_8 +MERLIN_PATH= +VENV_NAME= # Activate the virtual environment source ${MERLIN_PATH}/${VENV_NAME}/bin/activate From 536914feeea1706c9c943e86a511e7f43c75d369 Mon Sep 17 00:00:00 2001 From: Dylan Thomas Cliche Date: Mon, 4 Dec 2023 10:37:27 -0800 Subject: [PATCH 27/29] Took contact info out of README --- merlin/examples/workflows/stop_ensemble/README | 2 -- 1 file changed, 2 deletions(-) diff --git a/merlin/examples/workflows/stop_ensemble/README b/merlin/examples/workflows/stop_ensemble/README index 6a3eadd9c..30cddb6ac 100644 --- a/merlin/examples/workflows/stop_ensemble/README +++ b/merlin/examples/workflows/stop_ensemble/README @@ -10,8 +10,6 @@ | -hello_samples.yaml | | -quitScript.sh | | | -| Created By: Dylan Cliche (cliche1@llnl.gov) | -| Updated On: 08/14/2023 | +----------------------------------------------------+ From 581b7fa41e890daf259e33345d91b9e3cccd8387 Mon Sep 17 00:00:00 2001 From: Dylan Thomas Cliche Date: Mon, 4 Dec 2023 10:40:50 -0800 Subject: [PATCH 28/29] Updated CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9db50369a..61eda832a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Pytest fixtures in the `conftest.py` file of the integration test suite - NOTE: an export command `export LC_ALL='C'` had to be added to fix a bug in the WEAVE CI. This can be removed when we resolve this issue for the `merlin server` command - Tests for the `celeryadapter.py` module +- New example for stopping an ensemble workflow in `merlin/examples/workflows/stop_ensemble/`. ## [1.11.1] ### Fixed From 55ea1c0f5621f060bbdf1b4c36ff3d25cd81a589 Mon Sep 17 00:00:00 2001 From: Dylan Thomas Cliche Date: Fri, 22 Dec 2023 14:25:35 -0800 Subject: [PATCH 29/29] Fixed a bug. Where the would show up using scontrol show job doesn't show up anymore. This was remedied by checking the specification filename printed at the top of the output file. This also allowed for improved robustness by doing a whole match between the and the WorkDir set in scontrol show job . --- .../workflows/stop_ensemble/quitScript.sh | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/merlin/examples/workflows/stop_ensemble/quitScript.sh b/merlin/examples/workflows/stop_ensemble/quitScript.sh index 030650ac9..a94a5d3f6 100644 --- a/merlin/examples/workflows/stop_ensemble/quitScript.sh +++ b/merlin/examples/workflows/stop_ensemble/quitScript.sh @@ -3,7 +3,7 @@ #SBATCH -N 1 #SBATCH --ntasks-per-node=1 #SBATCH -J stopWorkers -#SBATCH -t 00:00:05 +#SBATCH -t 00:01:00 #SBATCH -o merlin_StopWorker_%j.out # Turn off core files to work around flux exec issue. @@ -18,14 +18,22 @@ do if [[ $JOB2CHECK != $SLURM_JOB_ID ]] then if [[ ! -z $(scontrol show job $JOB2CHECK | \ - grep "Command=$specRoot" | \ - grep -w $targetSpecFile) ]] + grep -w "WorkDir=$specRoot") ]] then - cancelJOB_NAME=$(squeue --me --job=$JOB2CHECK --format=%j | sed -n '2p') - echo $cancelJOB_NAME - echo "Job to Cancel::::Job Name: $cancelJOB_NAME Job ID: $JOB2CHECK" - echo "canceling job" - scancel $JOB2CHECK + outFile=$(scontrol show job $JOB2CHECK | \ + grep "StdOut=") + outFile=${outFile##*=} + + foundSpec=$(grep 'Specification File: ' $outFile | sed 's/^.*: //') + + if [[ $targetSpecFile == $foundSpec ]] + then + cancelJOB_NAME=$(squeue --me --job=$JOB2CHECK --format=%j | sed -n '2p') + echo $cancelJOB_NAME + echo "Job to Cancel::::Job Name: $cancelJOB_NAME Job ID: $JOB2CHECK" + echo "canceling job" + scancel $JOB2CHECK + fi fi fi done