From 282fb27bf523abeb12581b581ccbab0acdf5f37e Mon Sep 17 00:00:00 2001 From: burnout87 Date: Tue, 26 Mar 2024 15:23:55 +0100 Subject: [PATCH 01/41] config file --- nb2workflow/__init__.py | 1 + nb2workflow/configurer.py | 37 +++++++++++++++++++++++++++++++++++++ nb2workflow/service.py | 4 ++++ 3 files changed, 42 insertions(+) create mode 100644 nb2workflow/configurer.py diff --git a/nb2workflow/__init__.py b/nb2workflow/__init__.py index cd45dbc0..01da8385 100644 --- a/nb2workflow/__init__.py +++ b/nb2workflow/__init__.py @@ -1,6 +1,7 @@ import pkg_resources name = "nb2workflow" +conf_dir = 'config_dir' def version(): v = pkg_resources.get_distribution("nb2workflow").version diff --git a/nb2workflow/configurer.py b/nb2workflow/configurer.py new file mode 100644 index 00000000..2207e8b8 --- /dev/null +++ b/nb2workflow/configurer.py @@ -0,0 +1,37 @@ +import os +import yaml +import logging + +from nb2workflow import conf_dir + +logger = logging.getLogger("conf") + + +class ConfigEnv(object): + + def __init__(self, cfg_dict): + self.cfg_dict = cfg_dict + + if 'service' in self.cfg_dict.keys(): + service_dict = cfg_dict['service'] + + self.set_service_conf(service_dict.get('max_download_size', 1000000000)) + + @classmethod + def from_conf_file(cls, conf_file_path, set_by=None): + + if conf_file_path is None: + conf_file_path = os.path.join(conf_dir, 'service_conf.yml') + logger.info(f"using conf file from default dir {conf_file_path}") + else: + logger.info(f"loading config from the file: {conf_file_path}") + + with open(conf_file_path, 'r') as conf_file: + cfg_dict = yaml.load(conf_file, Loader=yaml.SafeLoader) + + logger.debug('cfg_dict: %s', cfg_dict) + return ConfigEnv(cfg_dict) + + def set_service_conf(self, + max_download_size): + self.max_download_size = max_download_size diff --git a/nb2workflow/service.py b/nb2workflow/service.py index 133b66a9..89b7b9d4 100644 --- a/nb2workflow/service.py +++ b/nb2workflow/service.py @@ -691,6 +691,7 @@ def main(): parser.add_argument('--debug', action="store_true") parser.add_argument('--one-shot', metavar='workflow', type=str) parser.add_argument('--pattern', type=str, default=r'.*') + parser.add_argument('-conf_file', type=str, default=None) args = parser.parse_args() @@ -736,6 +737,9 @@ def main(): async_worker = AsyncWorker('default-%i' % worker_i) async_worker.start() + conf_file = args.conf_file + conf = ConfigEnv.from_conf_file(conf_file, set_by=f'command line {__file__}:{__name__}') + app.run(host=args.host, port=args.port) From 536833d5518e2eabf24182b744ee09c6fd047a42 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 8 Apr 2024 16:39:28 +0200 Subject: [PATCH 02/41] import --- nb2workflow/service.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/nb2workflow/service.py b/nb2workflow/service.py index 89b7b9d4..e77092e9 100644 --- a/nb2workflow/service.py +++ b/nb2workflow/service.py @@ -4,6 +4,7 @@ from werkzeug.routing import RequestRedirect + try: from werkzeug.exceptions import MethodNotAllowed, NotFound except ImportError: @@ -13,6 +14,7 @@ import queue from nb2workflow import ontology, publish, schedule from nb2workflow.nbadapter import NotebookAdapter, find_notebooks, PapermillWorkflowIncomplete +from nb2workflow.configurer import ConfigEnv import os import json @@ -713,6 +715,11 @@ def main(): root.setLevel(logging.INFO) handler.setLevel(logging.INFO) + conf_file = args.conf_file + logger.info(f"loading config from {conf_file}") + conf = ConfigEnv.from_conf_file(conf_file, set_by=f'command line {__file__}:{__name__}') + app.config['conf'] = conf + app.notebook_adapters = find_notebooks(args.notebook, pattern=args.pattern) setup_routes(app) app.service_semantic_signature = ontology.service_semantic_signature( @@ -737,9 +744,6 @@ def main(): async_worker = AsyncWorker('default-%i' % worker_i) async_worker.start() - conf_file = args.conf_file - conf = ConfigEnv.from_conf_file(conf_file, set_by=f'command line {__file__}:{__name__}') - app.run(host=args.host, port=args.port) From 5cbcaba886302c5536e82f1f0a6267c26e8b56e3 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 8 Apr 2024 16:59:46 +0200 Subject: [PATCH 03/41] conf and conf_example --- nb2workflow/config_dir/conf_env.yml | 2 ++ nb2workflow/config_dir/conf_env.yml.example | 4 ++++ 2 files changed, 6 insertions(+) create mode 100644 nb2workflow/config_dir/conf_env.yml create mode 100644 nb2workflow/config_dir/conf_env.yml.example diff --git a/nb2workflow/config_dir/conf_env.yml b/nb2workflow/config_dir/conf_env.yml new file mode 100644 index 00000000..10c41374 --- /dev/null +++ b/nb2workflow/config_dir/conf_env.yml @@ -0,0 +1,2 @@ +service: + max_download_size: 1000000000 \ No newline at end of file diff --git a/nb2workflow/config_dir/conf_env.yml.example b/nb2workflow/config_dir/conf_env.yml.example new file mode 100644 index 00000000..f703a2f7 --- /dev/null +++ b/nb2workflow/config_dir/conf_env.yml.example @@ -0,0 +1,4 @@ +# example configuration, please adapt! + +service: + max_download_size: 1000000000 \ No newline at end of file From cb6947e335082ccc4ff1de0b01a4a8d9b8e00b5f Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 8 Apr 2024 17:42:17 +0200 Subject: [PATCH 04/41] default config --- nb2workflow/__init__.py | 3 ++- nb2workflow/configurer.py | 16 +++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/nb2workflow/__init__.py b/nb2workflow/__init__.py index 01da8385..809768f3 100644 --- a/nb2workflow/__init__.py +++ b/nb2workflow/__init__.py @@ -1,7 +1,8 @@ import pkg_resources +import os name = "nb2workflow" -conf_dir = 'config_dir' +conf_dir = os.path.dirname(__file__)+'/config_dir' def version(): v = pkg_resources.get_distribution("nb2workflow").version diff --git a/nb2workflow/configurer.py b/nb2workflow/configurer.py index 2207e8b8..2f81d43e 100644 --- a/nb2workflow/configurer.py +++ b/nb2workflow/configurer.py @@ -9,13 +9,15 @@ class ConfigEnv(object): - def __init__(self, cfg_dict): - self.cfg_dict = cfg_dict - - if 'service' in self.cfg_dict.keys(): - service_dict = cfg_dict['service'] - - self.set_service_conf(service_dict.get('max_download_size', 1000000000)) + def __init__(self, cfg_dict=None): + if cfg_dict is None: + cfg_dict = {} + service_cfg_dict = cfg_dict.get('service', {}) + if not service_cfg_dict: + max_download_size = 1000000000 + else: + max_download_size = service_cfg_dict.get('max_download_size', 1000000000) + self.set_service_conf(max_download_size=max_download_size) @classmethod def from_conf_file(cls, conf_file_path, set_by=None): From 853a62ef8bb92fc94471b6605564633cfe7a4f1c Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 8 Apr 2024 17:45:27 +0200 Subject: [PATCH 05/41] default config not from file --- nb2workflow/configurer.py | 9 ++------- nb2workflow/service.py | 8 ++++++-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/nb2workflow/configurer.py b/nb2workflow/configurer.py index 2f81d43e..24be4822 100644 --- a/nb2workflow/configurer.py +++ b/nb2workflow/configurer.py @@ -20,13 +20,8 @@ def __init__(self, cfg_dict=None): self.set_service_conf(max_download_size=max_download_size) @classmethod - def from_conf_file(cls, conf_file_path, set_by=None): - - if conf_file_path is None: - conf_file_path = os.path.join(conf_dir, 'service_conf.yml') - logger.info(f"using conf file from default dir {conf_file_path}") - else: - logger.info(f"loading config from the file: {conf_file_path}") + def from_conf_file(cls, conf_file_path): + logger.info(f"loading config from the file: {conf_file_path}") with open(conf_file_path, 'r') as conf_file: cfg_dict = yaml.load(conf_file, Loader=yaml.SafeLoader) diff --git a/nb2workflow/service.py b/nb2workflow/service.py index e77092e9..c7dd07d2 100644 --- a/nb2workflow/service.py +++ b/nb2workflow/service.py @@ -716,8 +716,12 @@ def main(): handler.setLevel(logging.INFO) conf_file = args.conf_file - logger.info(f"loading config from {conf_file}") - conf = ConfigEnv.from_conf_file(conf_file, set_by=f'command line {__file__}:{__name__}') + if conf_file is None: + logger.info(f"using default conf file from default") + conf = ConfigEnv() + else: + conf = ConfigEnv.from_conf_file(conf_file) + app.config['conf'] = conf app.notebook_adapters = find_notebooks(args.notebook, pattern=args.pattern) From 96b3c6868243c4935a4270867e386bb2e9a4a1a0 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 8 Apr 2024 18:08:05 +0200 Subject: [PATCH 06/41] string formatting --- nb2workflow/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nb2workflow/service.py b/nb2workflow/service.py index c7dd07d2..4b75854a 100644 --- a/nb2workflow/service.py +++ b/nb2workflow/service.py @@ -717,7 +717,7 @@ def main(): conf_file = args.conf_file if conf_file is None: - logger.info(f"using default conf file from default") + logger.info("using default conf file from default") conf = ConfigEnv() else: conf = ConfigEnv.from_conf_file(conf_file) From 01212003cff9ccbac1ea56c88bb55e4f44481454 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 8 Apr 2024 18:39:22 +0200 Subject: [PATCH 07/41] max_download_size nbadapter arg for init --- nb2workflow/nbadapter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nb2workflow/nbadapter.py b/nb2workflow/nbadapter.py index 08f2256a..f1d34f01 100644 --- a/nb2workflow/nbadapter.py +++ b/nb2workflow/nbadapter.py @@ -249,7 +249,7 @@ class NotebookAdapter: limit_output_attachment_file = None - def __init__(self, notebook_fn, tempdir_cache=None, n_download_max_tries=10, download_retry_sleep=.5): + def __init__(self, notebook_fn, tempdir_cache=None, n_download_max_tries=10, download_retry_sleep=.5, max_download_size=1e6): self.notebook_fn = os.path.abspath(notebook_fn) self.name = notebook_short_name(notebook_fn) self.tempdir_cache = tempdir_cache @@ -257,6 +257,7 @@ def __init__(self, notebook_fn, tempdir_cache=None, n_download_max_tries=10, dow logger.debug(self.extract_parameters()) self.n_download_max_tries = n_download_max_tries self.download_retry_sleep_s = download_retry_sleep + self.max_download_size = max_download_size @staticmethod def get_unique_filename_from_url(file_url): From 8a10a2c0f333821e4ac36da81784bb627d3a3d42 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Tue, 9 Apr 2024 16:57:41 +0200 Subject: [PATCH 08/41] using flaskdynaconf --- nb2workflow/config_dir/conf_env.yml | 2 -- nb2workflow/config_dir/conf_env.yml.example | 4 --- nb2workflow/configurer.py | 34 --------------------- nb2workflow/service.py | 26 ++++++++++------ settings.toml | 4 +++ setup.py | 3 +- 6 files changed, 23 insertions(+), 50 deletions(-) delete mode 100644 nb2workflow/config_dir/conf_env.yml delete mode 100644 nb2workflow/config_dir/conf_env.yml.example delete mode 100644 nb2workflow/configurer.py create mode 100644 settings.toml diff --git a/nb2workflow/config_dir/conf_env.yml b/nb2workflow/config_dir/conf_env.yml deleted file mode 100644 index 10c41374..00000000 --- a/nb2workflow/config_dir/conf_env.yml +++ /dev/null @@ -1,2 +0,0 @@ -service: - max_download_size: 1000000000 \ No newline at end of file diff --git a/nb2workflow/config_dir/conf_env.yml.example b/nb2workflow/config_dir/conf_env.yml.example deleted file mode 100644 index f703a2f7..00000000 --- a/nb2workflow/config_dir/conf_env.yml.example +++ /dev/null @@ -1,4 +0,0 @@ -# example configuration, please adapt! - -service: - max_download_size: 1000000000 \ No newline at end of file diff --git a/nb2workflow/configurer.py b/nb2workflow/configurer.py deleted file mode 100644 index 24be4822..00000000 --- a/nb2workflow/configurer.py +++ /dev/null @@ -1,34 +0,0 @@ -import os -import yaml -import logging - -from nb2workflow import conf_dir - -logger = logging.getLogger("conf") - - -class ConfigEnv(object): - - def __init__(self, cfg_dict=None): - if cfg_dict is None: - cfg_dict = {} - service_cfg_dict = cfg_dict.get('service', {}) - if not service_cfg_dict: - max_download_size = 1000000000 - else: - max_download_size = service_cfg_dict.get('max_download_size', 1000000000) - self.set_service_conf(max_download_size=max_download_size) - - @classmethod - def from_conf_file(cls, conf_file_path): - logger.info(f"loading config from the file: {conf_file_path}") - - with open(conf_file_path, 'r') as conf_file: - cfg_dict = yaml.load(conf_file, Loader=yaml.SafeLoader) - - logger.debug('cfg_dict: %s', cfg_dict) - return ConfigEnv(cfg_dict) - - def set_service_conf(self, - max_download_size): - self.max_download_size = max_download_size diff --git a/nb2workflow/service.py b/nb2workflow/service.py index 4b75854a..f26b6738 100644 --- a/nb2workflow/service.py +++ b/nb2workflow/service.py @@ -14,7 +14,6 @@ import queue from nb2workflow import ontology, publish, schedule from nb2workflow.nbadapter import NotebookAdapter, find_notebooks, PapermillWorkflowIncomplete -from nb2workflow.configurer import ConfigEnv import os import json @@ -38,6 +37,8 @@ from flask_caching import Cache from flask_cors import CORS +from dynaconf import FlaskDynaconf + from flasgger import LazyString, Swagger, swag_from from nb2workflow.workflows import serialize_workflow_exception @@ -94,6 +95,8 @@ def create_app(): "version": "0.0.1" } } + + FlaskDynaconf(app, settings_files=["settings.toml"]) swagger = Swagger(app, template=template) app.wsgi_app = ReverseProxied(app.wsgi_app) app.json_encoder = CustomJSONEncoder @@ -693,10 +696,15 @@ def main(): parser.add_argument('--debug', action="store_true") parser.add_argument('--one-shot', metavar='workflow', type=str) parser.add_argument('--pattern', type=str, default=r'.*') - parser.add_argument('-conf_file', type=str, default=None) + parser.add_argument('-s', '--settings', action="append", default=None) args = parser.parse_args() + if args.settings is not None: + for item in args.settings: + key, value = item.split('=') + app.config['SERVICE'][key] = value + handler = logging.StreamHandler() handler.setLevel(logging.INFO) @@ -715,14 +723,14 @@ def main(): root.setLevel(logging.INFO) handler.setLevel(logging.INFO) - conf_file = args.conf_file - if conf_file is None: - logger.info("using default conf file from default") - conf = ConfigEnv() - else: - conf = ConfigEnv.from_conf_file(conf_file) + # conf_file = args.conf_file + # if conf_file is None: + # logger.info("using default conf file from default") + # conf = ConfigEnv() + # else: + # conf = ConfigEnv.from_conf_file(conf_file) - app.config['conf'] = conf + # app.config['conf'] = conf app.notebook_adapters = find_notebooks(args.notebook, pattern=args.pattern) setup_routes(app) diff --git a/settings.toml b/settings.toml new file mode 100644 index 00000000..7cfff65f --- /dev/null +++ b/settings.toml @@ -0,0 +1,4 @@ +[default.service] +max_download_size = 1000000000 +n_download_max_tries = 10 +download_retry_sleep = 0.5 \ No newline at end of file diff --git a/setup.py b/setup.py index 7540e02a..bc781555 100644 --- a/setup.py +++ b/setup.py @@ -98,7 +98,8 @@ 'validators', 'sentry_sdk', 'rdflib', - 'GitPython' + 'GitPython', + 'dynaconf' ], From 28019da30fa72f7d004cf153fc8dcf5fe49d23d7 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Tue, 9 Apr 2024 17:48:14 +0200 Subject: [PATCH 09/41] passing config to find notebooks --- nb2workflow/nbadapter.py | 22 +++++++++++++++------- nb2workflow/service.py | 9 --------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/nb2workflow/nbadapter.py b/nb2workflow/nbadapter.py index f1d34f01..a38927b6 100644 --- a/nb2workflow/nbadapter.py +++ b/nb2workflow/nbadapter.py @@ -249,15 +249,15 @@ class NotebookAdapter: limit_output_attachment_file = None - def __init__(self, notebook_fn, tempdir_cache=None, n_download_max_tries=10, download_retry_sleep=.5, max_download_size=1e6): + def __init__(self, notebook_fn, tempdir_cache=None, n_download_max_tries=None, download_retry_sleep=None, max_download_size=None): self.notebook_fn = os.path.abspath(notebook_fn) self.name = notebook_short_name(notebook_fn) self.tempdir_cache = tempdir_cache logger.debug("notebook adapter for %s", self.notebook_fn) logger.debug(self.extract_parameters()) - self.n_download_max_tries = n_download_max_tries - self.download_retry_sleep_s = download_retry_sleep - self.max_download_size = max_download_size + self.n_download_max_tries = n_download_max_tries if n_download_max_tries is not None else 10 + self.download_retry_sleep_s = download_retry_sleep if download_retry_sleep is not None else .5 + self.max_download_size = max_download_size if max_download_size is not None else 1e6 @staticmethod def get_unique_filename_from_url(file_url): @@ -713,13 +713,15 @@ def remove_tmpdir(self): def notebook_short_name(ipynb_fn): return os.path.basename(ipynb_fn).replace(".ipynb","") -def find_notebooks(source, tests=False, pattern = r'.*') -> Dict[str, NotebookAdapter]: +def find_notebooks(source, tests=False, pattern = r'.*', config=None) -> Dict[str, NotebookAdapter]: def base_filter(fn): good = "output" not in fn and "preproc" not in fn good = good and re.match(pattern, os.path.basename(fn)) return good - + + if config is None: + config = dict() if tests: filt = lambda fn: base_filter(fn) and "/test_" in fn @@ -735,7 +737,13 @@ def base_filter(fn): raise Exception("no notebooks found in the directory:",source) notebook_adapters=dict([ - (notebook_short_name(notebook),NotebookAdapter(notebook)) for notebook in notebooks + ( + notebook_short_name(notebook), + NotebookAdapter(notebook, + n_download_max_tries=config.get('SERVICE.N_DOWNLOAD_MAX_TRIES', None), + download_retry_sleep=config.get('SERVICE.DOWNLOAD_RETRY_SLEEP', None), + max_download_size=config.get('SERVICE.MAX_DOWNLOAD_SIZE', None)) + ) for notebook in notebooks ]) logger.debug("notebook adapters: %s",notebook_adapters) diff --git a/nb2workflow/service.py b/nb2workflow/service.py index f26b6738..78fb1ddd 100644 --- a/nb2workflow/service.py +++ b/nb2workflow/service.py @@ -723,15 +723,6 @@ def main(): root.setLevel(logging.INFO) handler.setLevel(logging.INFO) - # conf_file = args.conf_file - # if conf_file is None: - # logger.info("using default conf file from default") - # conf = ConfigEnv() - # else: - # conf = ConfigEnv.from_conf_file(conf_file) - - # app.config['conf'] = conf - app.notebook_adapters = find_notebooks(args.notebook, pattern=args.pattern) setup_routes(app) app.service_semantic_signature = ontology.service_semantic_signature( From 9fe50e9e1f288e1e55c3b523cbc04638882ef30b Mon Sep 17 00:00:00 2001 From: burnout87 Date: Tue, 9 Apr 2024 17:52:59 +0200 Subject: [PATCH 10/41] passing config to find notebooks --- nb2workflow/nbadapter.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/nb2workflow/nbadapter.py b/nb2workflow/nbadapter.py index a38927b6..55c5bc21 100644 --- a/nb2workflow/nbadapter.py +++ b/nb2workflow/nbadapter.py @@ -722,6 +722,9 @@ def base_filter(fn): if config is None: config = dict() + n_download_max_tries = config.get('SERVICE.N_DOWNLOAD_MAX_TRIES', None), + download_retry_sleep = config.get('SERVICE.DOWNLOAD_RETRY_SLEEP', None), + max_download_size = config.get('SERVICE.MAX_DOWNLOAD_SIZE', None) if tests: filt = lambda fn: base_filter(fn) and "/test_" in fn @@ -740,9 +743,9 @@ def base_filter(fn): ( notebook_short_name(notebook), NotebookAdapter(notebook, - n_download_max_tries=config.get('SERVICE.N_DOWNLOAD_MAX_TRIES', None), - download_retry_sleep=config.get('SERVICE.DOWNLOAD_RETRY_SLEEP', None), - max_download_size=config.get('SERVICE.MAX_DOWNLOAD_SIZE', None)) + n_download_max_tries=n_download_max_tries, + download_retry_sleep=download_retry_sleep, + max_download_size=max_download_size) ) for notebook in notebooks ]) logger.debug("notebook adapters: %s",notebook_adapters) @@ -751,7 +754,10 @@ def base_filter(fn): elif os.path.isfile(source): if pattern != r'.*': logger.warning('Filename pattern is set but source %s is a single file. Ignoring pattern.') - notebook_adapters={notebook_short_name(source): NotebookAdapter(source)} + notebook_adapters={notebook_short_name(source): NotebookAdapter(source, + n_download_max_tries=n_download_max_tries, + download_retry_sleep=download_retry_sleep, + max_download_size=max_download_size)} else: raise Exception("requested notebook not found:",source) From de621d8a01a01d484e1174d364e8c4b6e1475bd1 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Tue, 9 Apr 2024 19:15:54 +0200 Subject: [PATCH 11/41] loggin test_Service --- tests/test_service.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_service.py b/tests/test_service.py index d1805188..e93c31d2 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -57,6 +57,8 @@ def test_service(client): for l in sorted(r.json, key=lambda x:x['ctime']): logger.info(l) + logger.info(f"r: {r.text}") + job = r.json[-1]['fn'].split("/")[-1] logger.info("job %s", job) From a63cbaddb4f02617dc88bc3018a76e377de7e829 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Tue, 9 Apr 2024 19:22:56 +0200 Subject: [PATCH 12/41] loggin test_Service --- tests/test_service.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_service.py b/tests/test_service.py index e93c31d2..e2b21970 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -57,7 +57,6 @@ def test_service(client): for l in sorted(r.json, key=lambda x:x['ctime']): logger.info(l) - logger.info(f"r: {r.text}") job = r.json[-1]['fn'].split("/")[-1] logger.info("job %s", job) From ea5f0aa464d1d7afa6f039e5abe6568457f25018 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Tue, 9 Apr 2024 19:54:01 +0200 Subject: [PATCH 13/41] check download_limit before download --- nb2workflow/nbadapter.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/nb2workflow/nbadapter.py b/nb2workflow/nbadapter.py index 55c5bc21..4415f569 100644 --- a/nb2workflow/nbadapter.py +++ b/nb2workflow/nbadapter.py @@ -28,6 +28,7 @@ import nbformat from nbconvert import HTMLExporter from urllib.parse import urlencode, urlparse +from urllib import request from . import logstash @@ -566,9 +567,20 @@ def extract_output_declarations(self): def extract_output(self): return self.extract_pm_output() - def download_file(self, file_url, tmpdir): + def download_file(self, file_url, tmpdir, download_limit): n_download_tries_left = self.n_download_max_tries file_name = NotebookAdapter.get_unique_filename_from_url(file_url) + req = request.Request('https://sra-download.ncbi.nlm.nih.gov/traces/sra46/SRR/005150/SRR5273887', + method='HEAD') + f = request.urlopen(req) + file_size = int(f.headers['Content-Length']) + if file_size > download_limit: + msg = (f"An issue occurred when attempting to download the url {file_url}, " + "the file appears to be too large to download, " + f"and the download limit is set to {download_limit} bytes.") + logger.warning(msg) + sentry.capture_message(msg) + raise Exception(msg) while True: response = requests.get(file_url) if response.status_code == 200: From 6f772634ca66cdc58a04cd5be50633d76ab2fd7f Mon Sep 17 00:00:00 2001 From: burnout87 Date: Tue, 9 Apr 2024 19:54:39 +0200 Subject: [PATCH 14/41] check download_limit before download --- nb2workflow/nbadapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nb2workflow/nbadapter.py b/nb2workflow/nbadapter.py index 4415f569..1e90e704 100644 --- a/nb2workflow/nbadapter.py +++ b/nb2workflow/nbadapter.py @@ -615,7 +615,7 @@ def extract_files_locally(self, parameters, tmpdir): if validators.url(arg_par_value): logger.debug(f'download {arg_par_value}') try: - file_name = self.download_file(arg_par_value, tmpdir) + file_name = self.download_file(arg_par_value, tmpdir, self.max_download_size) adapted_parameters[input_par_name] = file_name except Exception as e: exceptions.append(e) From a02325f1bbcf5dd975b3c8227b9d110675e3a61a Mon Sep 17 00:00:00 2001 From: burnout87 Date: Wed, 10 Apr 2024 08:28:09 +0200 Subject: [PATCH 15/41] using request lib --- nb2workflow/nbadapter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nb2workflow/nbadapter.py b/nb2workflow/nbadapter.py index 1e90e704..fd8890c8 100644 --- a/nb2workflow/nbadapter.py +++ b/nb2workflow/nbadapter.py @@ -570,10 +570,10 @@ def extract_output(self): def download_file(self, file_url, tmpdir, download_limit): n_download_tries_left = self.n_download_max_tries file_name = NotebookAdapter.get_unique_filename_from_url(file_url) - req = request.Request('https://sra-download.ncbi.nlm.nih.gov/traces/sra46/SRR/005150/SRR5273887', - method='HEAD') - f = request.urlopen(req) - file_size = int(f.headers['Content-Length']) + + response = requests.head(file_url) + file_size = int(response.headers.get('Content-Length', 0)) + if file_size > download_limit: msg = (f"An issue occurred when attempting to download the url {file_url}, " "the file appears to be too large to download, " From 145be50314db16aae3fcaca9a216381197df4922 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Wed, 10 Apr 2024 08:37:38 +0200 Subject: [PATCH 16/41] multiple attempts request infos file url --- nb2workflow/nbadapter.py | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/nb2workflow/nbadapter.py b/nb2workflow/nbadapter.py index fd8890c8..a85b0842 100644 --- a/nb2workflow/nbadapter.py +++ b/nb2workflow/nbadapter.py @@ -569,18 +569,33 @@ def extract_output(self): def download_file(self, file_url, tmpdir, download_limit): n_download_tries_left = self.n_download_max_tries + n_attempts_request_info = self.n_download_max_tries file_name = NotebookAdapter.get_unique_filename_from_url(file_url) - - response = requests.head(file_url) - file_size = int(response.headers.get('Content-Length', 0)) - - if file_size > download_limit: - msg = (f"An issue occurred when attempting to download the url {file_url}, " - "the file appears to be too large to download, " - f"and the download limit is set to {download_limit} bytes.") - logger.warning(msg) - sentry.capture_message(msg) - raise Exception(msg) + while True: + response = requests.head(file_url) + file_size = int(response.headers.get('Content-Length', 0)) + if response.status_code == 200: + if file_size > download_limit: + msg = (f"An issue occurred when attempting to download the url {file_url}, " + "the file appears to be too large to download, " + f"and the download limit is set to {download_limit} bytes.") + logger.warning(msg) + sentry.capture_message(msg) + raise Exception(msg) + break + else: + n_attempts_request_info -= 1 + if n_attempts_request_info > 0: + logger.warning( + f"An issue occurred when attempting to request information about the file at the url {file_url}, " + f"sleeping {self.download_retry_sleep_s} seconds until retry") + time.sleep(self.download_retry_sleep_s) + else: + msg = (f"An issue occurred when attempting to request information about file at the url {file_url}, " + "this might be related to an invalid url, please check the input provided") + logger.warning(msg) + sentry.capture_message(msg) + raise Exception(msg) while True: response = requests.get(file_url) if response.status_code == 200: From ada5c008cf80d8dd2794960aa5b32057b06b602e Mon Sep 17 00:00:00 2001 From: burnout87 Date: Wed, 10 Apr 2024 09:36:53 +0200 Subject: [PATCH 17/41] code optimized --- nb2workflow/nbadapter.py | 84 ++++++++++++++++++++-------------------- 1 file changed, 43 insertions(+), 41 deletions(-) diff --git a/nb2workflow/nbadapter.py b/nb2workflow/nbadapter.py index a85b0842..b47417b1 100644 --- a/nb2workflow/nbadapter.py +++ b/nb2workflow/nbadapter.py @@ -570,51 +570,53 @@ def extract_output(self): def download_file(self, file_url, tmpdir, download_limit): n_download_tries_left = self.n_download_max_tries n_attempts_request_info = self.n_download_max_tries + size_ok = False file_name = NotebookAdapter.get_unique_filename_from_url(file_url) while True: - response = requests.head(file_url) - file_size = int(response.headers.get('Content-Length', 0)) - if response.status_code == 200: - if file_size > download_limit: - msg = (f"An issue occurred when attempting to download the url {file_url}, " - "the file appears to be too large to download, " - f"and the download limit is set to {download_limit} bytes.") - logger.warning(msg) - sentry.capture_message(msg) - raise Exception(msg) - break - else: - n_attempts_request_info -= 1 - if n_attempts_request_info > 0: - logger.warning( - f"An issue occurred when attempting to request information about the file at the url {file_url}, " - f"sleeping {self.download_retry_sleep_s} seconds until retry") - time.sleep(self.download_retry_sleep_s) + if not size_ok: + response = requests.head(file_url) + if response.status_code == 200: + file_size = int(response.headers.get('Content-Length', 0)) + if file_size > download_limit: + msg = (f"An issue occurred when attempting to download the url {file_url}, " + "the file appears to be too large to download, " + f"and the download limit is set to {download_limit} bytes.") + logger.warning(msg) + sentry.capture_message(msg) + raise Exception(msg) + size_ok = True else: - msg = (f"An issue occurred when attempting to request information about file at the url {file_url}, " - "this might be related to an invalid url, please check the input provided") - logger.warning(msg) - sentry.capture_message(msg) - raise Exception(msg) - while True: - response = requests.get(file_url) - if response.status_code == 200: - with open(os.path.join(tmpdir, file_name), 'wb') as file: - file.write(response.content) - break - else: - n_download_tries_left -= 1 - if n_download_tries_left > 0: - logger.warning( - f"An issue occurred when attempting to download the file at the url {file_url}, " - f"sleeping {self.download_retry_sleep_s} seconds until retry") - time.sleep(self.download_retry_sleep_s) + n_attempts_request_info -= 1 + if n_attempts_request_info > 0: + logger.warning( + f"An issue occurred when attempting to request information about the file at the url {file_url}, " + f"sleeping {self.download_retry_sleep_s} seconds until retry") + time.sleep(self.download_retry_sleep_s) + else: + msg = (f"An issue occurred when attempting to request information about file at the url {file_url}, " + "this might be related to an invalid url, please check the input provided") + logger.warning(msg) + sentry.capture_message(msg) + raise Exception(msg) + if size_ok: + response = requests.get(file_url) + if response.status_code == 200: + with open(os.path.join(tmpdir, file_name), 'wb') as file: + file.write(response.content) + break else: - msg = (f"An issue occurred when attempting to download the url {file_url}, " - "this might be related to an invalid url, please check the input provided") - logger.warning(msg) - sentry.capture_message(msg) - raise Exception(msg) + n_download_tries_left -= 1 + if n_download_tries_left > 0: + logger.warning( + f"An issue occurred when attempting to download the file at the url {file_url}, " + f"sleeping {self.download_retry_sleep_s} seconds until retry") + time.sleep(self.download_retry_sleep_s) + else: + msg = (f"An issue occurred when attempting to download the url {file_url}, " + "this might be related to an invalid url, please check the input provided") + logger.warning(msg) + sentry.capture_message(msg) + raise Exception(msg) return file_name From 8e9491d7210a2a4cc84be3010500955692cbe5ab Mon Sep 17 00:00:00 2001 From: burnout87 Date: Wed, 10 Apr 2024 10:43:22 +0200 Subject: [PATCH 18/41] code optimized --- nb2workflow/nbadapter.py | 74 +++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 39 deletions(-) diff --git a/nb2workflow/nbadapter.py b/nb2workflow/nbadapter.py index b47417b1..323b0b27 100644 --- a/nb2workflow/nbadapter.py +++ b/nb2workflow/nbadapter.py @@ -569,54 +569,50 @@ def extract_output(self): def download_file(self, file_url, tmpdir, download_limit): n_download_tries_left = self.n_download_max_tries - n_attempts_request_info = self.n_download_max_tries size_ok = False + size_too_large = False file_name = NotebookAdapter.get_unique_filename_from_url(file_url) + file_path = os.path.join(tmpdir, file_name) while True: - if not size_ok: - response = requests.head(file_url) - if response.status_code == 200: - file_size = int(response.headers.get('Content-Length', 0)) - if file_size > download_limit: - msg = (f"An issue occurred when attempting to download the url {file_url}, " - "the file appears to be too large to download, " - f"and the download limit is set to {download_limit} bytes.") - logger.warning(msg) - sentry.capture_message(msg) - raise Exception(msg) - size_ok = True - else: - n_attempts_request_info -= 1 - if n_attempts_request_info > 0: - logger.warning( - f"An issue occurred when attempting to request information about the file at the url {file_url}, " - f"sleeping {self.download_retry_sleep_s} seconds until retry") - time.sleep(self.download_retry_sleep_s) + try: + step = 'getting file size' + if not size_ok: + response = requests.head(file_url) + if response.status_code == 200: + file_size = int(response.headers.get('Content-Length', 0)) + if file_size > download_limit: + size_too_large = True + msg = ("The file appears to be too large to download, " + f"and the download limit is set to {download_limit} bytes.") + logger.warning(msg) + sentry.capture_message(msg) + raise Exception(msg) else: - msg = (f"An issue occurred when attempting to request information about file at the url {file_url}, " - "this might be related to an invalid url, please check the input provided") - logger.warning(msg) - sentry.capture_message(msg) - raise Exception(msg) - if size_ok: + raise + size_ok = True + step = 'downloading file' response = requests.get(file_url) if response.status_code == 200: - with open(os.path.join(tmpdir, file_name), 'wb') as file: + with open(file_path, 'wb') as file: file.write(response.content) break else: - n_download_tries_left -= 1 - if n_download_tries_left > 0: - logger.warning( - f"An issue occurred when attempting to download the file at the url {file_url}, " - f"sleeping {self.download_retry_sleep_s} seconds until retry") - time.sleep(self.download_retry_sleep_s) - else: - msg = (f"An issue occurred when attempting to download the url {file_url}, " - "this might be related to an invalid url, please check the input provided") - logger.warning(msg) - sentry.capture_message(msg) - raise Exception(msg) + raise + except Exception as e: + if size_too_large: + raise e + n_download_tries_left -= 1 + if n_download_tries_left > 0: + logger.warning( + f"An issue occurred when attempting to {step} the file at the url {file_url}:\n{str(e)}\n" + f"Sleeping {self.download_retry_sleep_s} seconds until retry") + time.sleep(self.download_retry_sleep_s) + else: + msg = (f"An issue occurred when attempting to {step} the url {file_url}:\n{str(e)}\n" + "this might be related to an invalid url, please check the input provided") + logger.warning(msg) + sentry.capture_message(msg) + raise Exception(msg) return file_name From 60b7842cebc520071ce745c943a2e8377cb10847 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Wed, 10 Apr 2024 11:06:02 +0200 Subject: [PATCH 19/41] code optimized --- nb2workflow/nbadapter.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nb2workflow/nbadapter.py b/nb2workflow/nbadapter.py index 323b0b27..32910bb8 100644 --- a/nb2workflow/nbadapter.py +++ b/nb2workflow/nbadapter.py @@ -604,11 +604,12 @@ def download_file(self, file_url, tmpdir, download_limit): n_download_tries_left -= 1 if n_download_tries_left > 0: logger.warning( - f"An issue occurred when attempting to {step} the file at the url {file_url}:\n{str(e)}\n" - f"Sleeping {self.download_retry_sleep_s} seconds until retry") + (f"An issue occurred when attempting to {step} the file at the url {file_url}. " + f"Sleeping {self.download_retry_sleep_s} seconds until retry") + ) time.sleep(self.download_retry_sleep_s) else: - msg = (f"An issue occurred when attempting to {step} the url {file_url}:\n{str(e)}\n" + msg = (f"An issue occurred when attempting to {step} the url {file_url}." "this might be related to an invalid url, please check the input provided") logger.warning(msg) sentry.capture_message(msg) From 70f614649a824cab3f140e0759bedac788859b87 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Wed, 10 Apr 2024 11:32:34 +0200 Subject: [PATCH 20/41] code optimized --- nb2workflow/nbadapter.py | 72 ++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/nb2workflow/nbadapter.py b/nb2workflow/nbadapter.py index 32910bb8..f6d583dd 100644 --- a/nb2workflow/nbadapter.py +++ b/nb2workflow/nbadapter.py @@ -570,50 +570,50 @@ def extract_output(self): def download_file(self, file_url, tmpdir, download_limit): n_download_tries_left = self.n_download_max_tries size_ok = False - size_too_large = False + file_downloaded = False file_name = NotebookAdapter.get_unique_filename_from_url(file_url) file_path = os.path.join(tmpdir, file_name) - while True: - try: - step = 'getting file size' - if not size_ok: - response = requests.head(file_url) - if response.status_code == 200: - file_size = int(response.headers.get('Content-Length', 0)) - if file_size > download_limit: - size_too_large = True - msg = ("The file appears to be too large to download, " - f"and the download limit is set to {download_limit} bytes.") - logger.warning(msg) - sentry.capture_message(msg) - raise Exception(msg) - else: - raise - size_ok = True - step = 'downloading file' - response = requests.get(file_url) + for _ in range(n_download_tries_left): + step = 'getting the file size' + if not size_ok: + response = requests.head(file_url) if response.status_code == 200: - with open(file_path, 'wb') as file: - file.write(response.content) - break + file_size = int(response.headers.get('Content-Length', 0)) + if file_size > download_limit: + msg = ("The file appears to be too large to download, " + f"and the download limit is set to {download_limit} bytes.") + logger.warning(msg) + sentry.capture_message(msg) + raise Exception(msg) else: - raise - except Exception as e: - if size_too_large: - raise e - n_download_tries_left -= 1 - if n_download_tries_left > 0: logger.warning( - (f"An issue occurred when attempting to {step} the file at the url {file_url}. " + (f"An issue occurred when attempting to {step} of the file at the url {file_url}. " f"Sleeping {self.download_retry_sleep_s} seconds until retry") ) time.sleep(self.download_retry_sleep_s) - else: - msg = (f"An issue occurred when attempting to {step} the url {file_url}." - "this might be related to an invalid url, please check the input provided") - logger.warning(msg) - sentry.capture_message(msg) - raise Exception(msg) + continue + size_ok = True + step = 'downloading file' + response = requests.get(file_url) + if response.status_code == 200: + with open(file_path, 'wb') as file: + file.write(response.content) + file_downloaded = True + break + else: + logger.warning( + (f"An issue occurred when attempting to {step} the file at the url {file_url}. " + f"Sleeping {self.download_retry_sleep_s} seconds until retry") + ) + time.sleep(self.download_retry_sleep_s) + continue + + if not (file_downloaded and size_ok): + msg = (f"An issue occurred when attempting to {step} at the url {file_url}. " + "This might be related to an invalid url, please check the input provided") + logger.warning(msg) + sentry.capture_message(msg) + raise Exception(msg) return file_name From a0da3f403590fcaa96e61a95adfd48acd1122475 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Wed, 10 Apr 2024 11:32:40 +0200 Subject: [PATCH 21/41] test adapted --- tests/test_input_types.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_input_types.py b/tests/test_input_types.py index 34eada36..9e691c6d 100644 --- a/tests/test_input_types.py +++ b/tests/test_input_types.py @@ -22,8 +22,8 @@ def test_posix_download_file_with_arg(client): def test_posix_download_file_with_arg_wrong_url(client): r = client.get('/api/v1.0/get/testposixpath', query_string={'fits_file_path': 'https://fits.gsfc.nasa.gov/samples/aaaaaa.fits'}) - assert r.json['exceptions'][0] == ("Exception('An issue occurred when attempting to download the url " - "https://fits.gsfc.nasa.gov/samples/aaaaaa.fits, this might be related " + assert r.json['exceptions'][0] == ("Exception('An issue occurred when attempting to getting the file size at the url " + "https://fits.gsfc.nasa.gov/samples/aaaaaa.fits. This might be related " "to an invalid url, please check the input provided')") def test_boolean_default(client): From 420191babf80a62fbba0e660ba80023c2993b3a3 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Wed, 10 Apr 2024 16:06:27 +0200 Subject: [PATCH 22/41] providing config arg to NotebookAdapter init --- nb2workflow/nbadapter.py | 4 ++-- nb2workflow/service.py | 22 +++++++++++++++++++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/nb2workflow/nbadapter.py b/nb2workflow/nbadapter.py index f6d583dd..c9e71ff8 100644 --- a/nb2workflow/nbadapter.py +++ b/nb2workflow/nbadapter.py @@ -748,8 +748,8 @@ def base_filter(fn): if config is None: config = dict() - n_download_max_tries = config.get('SERVICE.N_DOWNLOAD_MAX_TRIES', None), - download_retry_sleep = config.get('SERVICE.DOWNLOAD_RETRY_SLEEP', None), + n_download_max_tries = config.get('SERVICE.N_DOWNLOAD_MAX_TRIES', None) + download_retry_sleep = config.get('SERVICE.DOWNLOAD_RETRY_SLEEP', None) max_download_size = config.get('SERVICE.MAX_DOWNLOAD_SIZE', None) if tests: diff --git a/nb2workflow/service.py b/nb2workflow/service.py index 78fb1ddd..0dff1c54 100644 --- a/nb2workflow/service.py +++ b/nb2workflow/service.py @@ -177,7 +177,15 @@ def _run(self): return template_nba = app.notebook_adapters.get(self.target) - nba = NotebookAdapter(template_nba.notebook_fn, tempdir_cache=app.async_workflow_jobdirs) + + n_download_max_tries = app.config.get('SERVICE.N_DOWNLOAD_MAX_TRIES', None) + download_retry_sleep = app.config.get('SERVICE.DOWNLOAD_RETRY_SLEEP', None) + max_download_size = app.config.get('SERVICE.MAX_DOWNLOAD_SIZE', None) + + nba = NotebookAdapter(template_nba.notebook_fn, tempdir_cache=app.async_workflow_jobdirs, + n_download_max_tries=n_download_max_tries, + download_retry_sleep=download_retry_sleep, + max_download_size=max_download_size) app.async_workflows[self.key] = 'started' self.perform_callback(action='progress') @@ -269,7 +277,15 @@ def workflow(target, background=False, async_request=False): logger.debug("raw parameters %s", request.args) template_nba = app.notebook_adapters.get(target) - nba = NotebookAdapter(template_nba.notebook_fn) + + n_download_max_tries = app.config.get('SERVICE.N_DOWNLOAD_MAX_TRIES', None) + download_retry_sleep = app.config.get('SERVICE.DOWNLOAD_RETRY_SLEEP', None) + max_download_size = app.config.get('SERVICE.MAX_DOWNLOAD_SIZE', None) + + nba = NotebookAdapter(template_nba.notebook_fn, + n_download_max_tries=n_download_max_tries, + download_retry_sleep=download_retry_sleep, + max_download_size=max_download_size) if nba is None: interpreted_parameters = None @@ -723,7 +739,7 @@ def main(): root.setLevel(logging.INFO) handler.setLevel(logging.INFO) - app.notebook_adapters = find_notebooks(args.notebook, pattern=args.pattern) + app.notebook_adapters = find_notebooks(args.notebook, pattern=args.pattern, config=app.config) setup_routes(app) app.service_semantic_signature = ontology.service_semantic_signature( app.notebook_adapters) From f91058a7ce8176675f3a3d85097705fd8b77ac85 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Wed, 10 Apr 2024 16:06:35 +0200 Subject: [PATCH 23/41] adapted test --- tests/conftest.py | 12 ++++++++++++ tests/test_input_types.py | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 87debf7c..55f16681 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -46,6 +46,7 @@ def test_notebook_lfs_repo(): return path + @pytest.fixture def app(test_notebook): app = nb2workflow.service.app @@ -55,6 +56,17 @@ def app(test_notebook): return app +@pytest.fixture +def app_low_download_limit(app): + testfiles_path = os.path.join(os.path.dirname(__file__), 'testfiles') + app = nb2workflow.service.app + app.config['SERVICE.MAX_DOWNLOAD_SIZE'] = 1 + app.notebook_adapters = nb2workflow.nbadapter.find_notebooks(testfiles_path, config=app.config) + nb2workflow.service.setup_routes(app) + print("creating app with low limit on the download of files") + return app + + def kill_child_processes(parent_pid, sig=signal.SIGTERM): try: parent = psutil.Process(parent_pid) diff --git a/tests/test_input_types.py b/tests/test_input_types.py index 9e691c6d..d8520669 100644 --- a/tests/test_input_types.py +++ b/tests/test_input_types.py @@ -20,6 +20,11 @@ def test_posix_download_file_with_arg(client): r = client.get('/api/v1.0/get/testposixpath', query_string={'fits_file_path': 'https://fits.gsfc.nasa.gov/samples/testkeys.fits'}) assert r.json['output']['output_file_download'] == 'file downloaded successfully' +def test_posix_download_file_with_arg_low_download_limit(client, app_low_download_limit): + print(app_low_download_limit.config) + r = client.get('/api/v1.0/get/testposixpath', query_string={'fits_file_path': 'https://fits.gsfc.nasa.gov/samples/testkeys.fits'}) + assert r.json['output']['output_file_download'] == 'file not downloaded' + def test_posix_download_file_with_arg_wrong_url(client): r = client.get('/api/v1.0/get/testposixpath', query_string={'fits_file_path': 'https://fits.gsfc.nasa.gov/samples/aaaaaa.fits'}) assert r.json['exceptions'][0] == ("Exception('An issue occurred when attempting to getting the file size at the url " From 769c79526593ca866ef4a1eeeb1fbf48758bd7a3 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Wed, 17 Apr 2024 11:12:20 +0200 Subject: [PATCH 24/41] missing import --- nb2workflow/nbadapter.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/nb2workflow/nbadapter.py b/nb2workflow/nbadapter.py index c9e71ff8..70924a69 100644 --- a/nb2workflow/nbadapter.py +++ b/nb2workflow/nbadapter.py @@ -29,9 +29,9 @@ from nbconvert import HTMLExporter from urllib.parse import urlencode, urlparse from urllib import request +from dynaconf import Dynaconf from . import logstash - from nb2workflow.sentry import sentry from nb2workflow.health import current_health from nb2workflow import workflows @@ -790,8 +790,8 @@ def base_filter(fn): return notebook_adapters -def nbinspect(nb_source, out=True, machine_readable=False): - nbas = find_notebooks(nb_source) +def nbinspect(nb_source, out=True, machine_readable=False, config=None): + nbas = find_notebooks(nb_source, config=config) # class CustomEncoder(json.JSONEncoder): # def default(self, obj): @@ -926,9 +926,9 @@ def json(): -def nbrun(nb_source, inp, inplace=False, optional_dispather=True, machine_readable=False): +def nbrun(nb_source, inp, inplace=False, optional_dispather=True, machine_readable=False, config=None): - nbas = find_notebooks(nb_source) + nbas = find_notebooks(nb_source, config=config) if len(nbas) > 1: nba = nbas[inp.pop('notebook')] @@ -1052,7 +1052,9 @@ def main_inspect(): setup_logging(args.debug) - nbinspect(args.notebook, machine_readable=args.machine_readable) + config = Dynaconf(settings_files=['settings.toml']) + + nbinspect(args.notebook, machine_readable=args.machine_readable, config=config) def main(): @@ -1075,7 +1077,13 @@ def main(): setup_logging(args.debug) - nbrun(args.notebook, inputs, inplace=args.inplace, optional_dispather=not args.mmoda_validation, machine_readable=args.machine_readable) + config = Dynaconf(settings_files=['settings.toml']) + + nbrun(args.notebook, inputs, + inplace=args.inplace, + optional_dispather=not args.mmoda_validation, + machine_readable=args.machine_readable, + config=config) if __name__ == "__main__": From bae3e9e69992897f5aafa8e323fbfc3cabba9f7e Mon Sep 17 00:00:00 2001 From: burnout87 Date: Wed, 17 Apr 2024 11:17:05 +0200 Subject: [PATCH 25/41] passing config to NotebookAdapter --- nb2workflow/nbadapter.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/nb2workflow/nbadapter.py b/nb2workflow/nbadapter.py index 70924a69..42efdf35 100644 --- a/nb2workflow/nbadapter.py +++ b/nb2workflow/nbadapter.py @@ -250,15 +250,15 @@ class NotebookAdapter: limit_output_attachment_file = None - def __init__(self, notebook_fn, tempdir_cache=None, n_download_max_tries=None, download_retry_sleep=None, max_download_size=None): + def __init__(self, notebook_fn, tempdir_cache=None, config=None): self.notebook_fn = os.path.abspath(notebook_fn) self.name = notebook_short_name(notebook_fn) self.tempdir_cache = tempdir_cache logger.debug("notebook adapter for %s", self.notebook_fn) logger.debug(self.extract_parameters()) - self.n_download_max_tries = n_download_max_tries if n_download_max_tries is not None else 10 - self.download_retry_sleep_s = download_retry_sleep if download_retry_sleep is not None else .5 - self.max_download_size = max_download_size if max_download_size is not None else 1e6 + self.n_download_max_tries = config.get('SERVICE.N_DOWNLOAD_MAX_TRIES', 10) + self.download_retry_sleep = config.get('SERVICE.DOWNLOAD_RETRY_SLEEP', .5) + self.max_download_size = config.get('SERVICE.MAX_DOWNLOAD_SIZE', 1e6) @staticmethod def get_unique_filename_from_url(file_url): @@ -748,9 +748,6 @@ def base_filter(fn): if config is None: config = dict() - n_download_max_tries = config.get('SERVICE.N_DOWNLOAD_MAX_TRIES', None) - download_retry_sleep = config.get('SERVICE.DOWNLOAD_RETRY_SLEEP', None) - max_download_size = config.get('SERVICE.MAX_DOWNLOAD_SIZE', None) if tests: filt = lambda fn: base_filter(fn) and "/test_" in fn @@ -768,10 +765,7 @@ def base_filter(fn): notebook_adapters=dict([ ( notebook_short_name(notebook), - NotebookAdapter(notebook, - n_download_max_tries=n_download_max_tries, - download_retry_sleep=download_retry_sleep, - max_download_size=max_download_size) + NotebookAdapter(notebook, config=config) ) for notebook in notebooks ]) logger.debug("notebook adapters: %s",notebook_adapters) @@ -780,10 +774,7 @@ def base_filter(fn): elif os.path.isfile(source): if pattern != r'.*': logger.warning('Filename pattern is set but source %s is a single file. Ignoring pattern.') - notebook_adapters={notebook_short_name(source): NotebookAdapter(source, - n_download_max_tries=n_download_max_tries, - download_retry_sleep=download_retry_sleep, - max_download_size=max_download_size)} + notebook_adapters={notebook_short_name(source): NotebookAdapter(source, config=config)} else: raise Exception("requested notebook not found:",source) From c64856383465ad7dffab50cf3a6c79470326bae2 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Wed, 17 Apr 2024 11:22:05 +0200 Subject: [PATCH 26/41] no need to pass arg to download_file --- nb2workflow/nbadapter.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nb2workflow/nbadapter.py b/nb2workflow/nbadapter.py index 42efdf35..b001a3a9 100644 --- a/nb2workflow/nbadapter.py +++ b/nb2workflow/nbadapter.py @@ -257,7 +257,7 @@ def __init__(self, notebook_fn, tempdir_cache=None, config=None): logger.debug("notebook adapter for %s", self.notebook_fn) logger.debug(self.extract_parameters()) self.n_download_max_tries = config.get('SERVICE.N_DOWNLOAD_MAX_TRIES', 10) - self.download_retry_sleep = config.get('SERVICE.DOWNLOAD_RETRY_SLEEP', .5) + self.download_retry_sleep_s = config.get('SERVICE.DOWNLOAD_RETRY_SLEEP', .5) self.max_download_size = config.get('SERVICE.MAX_DOWNLOAD_SIZE', 1e6) @staticmethod @@ -567,7 +567,7 @@ def extract_output_declarations(self): def extract_output(self): return self.extract_pm_output() - def download_file(self, file_url, tmpdir, download_limit): + def download_file(self, file_url, tmpdir): n_download_tries_left = self.n_download_max_tries size_ok = False file_downloaded = False @@ -579,9 +579,9 @@ def download_file(self, file_url, tmpdir, download_limit): response = requests.head(file_url) if response.status_code == 200: file_size = int(response.headers.get('Content-Length', 0)) - if file_size > download_limit: + if file_size > self.max_download_size: msg = ("The file appears to be too large to download, " - f"and the download limit is set to {download_limit} bytes.") + f"and the download limit is set to {self.max_download_size} bytes.") logger.warning(msg) sentry.capture_message(msg) raise Exception(msg) @@ -629,7 +629,7 @@ def extract_files_locally(self, parameters, tmpdir): if validators.url(arg_par_value): logger.debug(f'download {arg_par_value}') try: - file_name = self.download_file(arg_par_value, tmpdir, self.max_download_size) + file_name = self.download_file(arg_par_value, tmpdir) adapted_parameters[input_par_name] = file_name except Exception as e: exceptions.append(e) From 402ce9a087834eb7ce0b2cf5367c135b1388e9a5 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Wed, 17 Apr 2024 11:23:06 +0200 Subject: [PATCH 27/41] no need for conf_dir --- nb2workflow/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nb2workflow/__init__.py b/nb2workflow/__init__.py index 809768f3..cc571161 100644 --- a/nb2workflow/__init__.py +++ b/nb2workflow/__init__.py @@ -2,7 +2,6 @@ import os name = "nb2workflow" -conf_dir = os.path.dirname(__file__)+'/config_dir' def version(): v = pkg_resources.get_distribution("nb2workflow").version From 8932525c7db738be9c2aed9b280c8dcab6de3c94 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Wed, 17 Apr 2024 11:35:18 +0200 Subject: [PATCH 28/41] init NotebookAdapter --- nb2workflow/service.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/nb2workflow/service.py b/nb2workflow/service.py index 0dff1c54..21496ecf 100644 --- a/nb2workflow/service.py +++ b/nb2workflow/service.py @@ -178,14 +178,7 @@ def _run(self): template_nba = app.notebook_adapters.get(self.target) - n_download_max_tries = app.config.get('SERVICE.N_DOWNLOAD_MAX_TRIES', None) - download_retry_sleep = app.config.get('SERVICE.DOWNLOAD_RETRY_SLEEP', None) - max_download_size = app.config.get('SERVICE.MAX_DOWNLOAD_SIZE', None) - - nba = NotebookAdapter(template_nba.notebook_fn, tempdir_cache=app.async_workflow_jobdirs, - n_download_max_tries=n_download_max_tries, - download_retry_sleep=download_retry_sleep, - max_download_size=max_download_size) + nba = NotebookAdapter(template_nba.notebook_fn, tempdir_cache=app.async_workflow_jobdirs, config=app.config) app.async_workflows[self.key] = 'started' self.perform_callback(action='progress') @@ -278,14 +271,7 @@ def workflow(target, background=False, async_request=False): template_nba = app.notebook_adapters.get(target) - n_download_max_tries = app.config.get('SERVICE.N_DOWNLOAD_MAX_TRIES', None) - download_retry_sleep = app.config.get('SERVICE.DOWNLOAD_RETRY_SLEEP', None) - max_download_size = app.config.get('SERVICE.MAX_DOWNLOAD_SIZE', None) - - nba = NotebookAdapter(template_nba.notebook_fn, - n_download_max_tries=n_download_max_tries, - download_retry_sleep=download_retry_sleep, - max_download_size=max_download_size) + nba = NotebookAdapter(template_nba.notebook_fn, config=app.config) if nba is None: interpreted_parameters = None From 4cf68f2bdd0330708e47d9bfdb89e819223c1014 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Wed, 17 Apr 2024 12:22:30 +0200 Subject: [PATCH 29/41] sentry_url in the config --- nb2workflow/nbadapter.py | 1 + nb2workflow/sentry.py | 4 ++++ nb2workflow/workflows.py | 6 +++++- settings.toml | 3 ++- 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/nb2workflow/nbadapter.py b/nb2workflow/nbadapter.py index b001a3a9..cc9a91cf 100644 --- a/nb2workflow/nbadapter.py +++ b/nb2workflow/nbadapter.py @@ -259,6 +259,7 @@ def __init__(self, notebook_fn, tempdir_cache=None, config=None): self.n_download_max_tries = config.get('SERVICE.N_DOWNLOAD_MAX_TRIES', 10) self.download_retry_sleep_s = config.get('SERVICE.DOWNLOAD_RETRY_SLEEP', .5) self.max_download_size = config.get('SERVICE.MAX_DOWNLOAD_SIZE', 1e6) + sentry.sentry_url = config.get('SERVICE.SENTRY_URL', None) @staticmethod def get_unique_filename_from_url(file_url): diff --git a/nb2workflow/sentry.py b/nb2workflow/sentry.py index 98647ad5..a5e56ef1 100644 --- a/nb2workflow/sentry.py +++ b/nb2workflow/sentry.py @@ -20,6 +20,10 @@ def sentry_url(self): self._sentry_url = os.environ.get('SENTRY_URL', "https://63ae106793010d836c74830fa75b300c@o264756.ingest.sentry.io/4506186624335872") return self._sentry_url + @sentry_url.setter + def sentry_url(self, url): + self._sentry_url = url + @property def have_sentry(self): if self.sentry_url is None or self.sentry_url == '' or sentry_sdk is None: diff --git a/nb2workflow/workflows.py b/nb2workflow/workflows.py index e32a8fd5..aa35348f 100644 --- a/nb2workflow/workflows.py +++ b/nb2workflow/workflows.py @@ -12,6 +12,8 @@ from nb2workflow import nbadapter +from dynaconf import Dynaconf + cache = Cache('.nb2workflow/cache') enable_cache = False @@ -58,6 +60,8 @@ def evaluate(router, *args, **kwargs): print("async_request is not used here, but is set to", async_request) + config = Dynaconf(settings_files=['settings.toml']) + sentry.sentry_url = config.default.service.sentry_url logstasher.set_context(dict(router=router, args=args, kwargs=kwargs)) logstasher.log(dict(event='starting')) @@ -80,7 +84,7 @@ def evaluate(router, *args, **kwargs): location = args[0] args = args[1:] - nba = nbadapter.NotebookAdapter(location+"/%s.ipynb"%args[0]) + nba = nbadapter.NotebookAdapter(location+"/%s.ipynb"%args[0], config=config) # unused args diff --git a/settings.toml b/settings.toml index 7cfff65f..b254dbd2 100644 --- a/settings.toml +++ b/settings.toml @@ -1,4 +1,5 @@ [default.service] max_download_size = 1000000000 n_download_max_tries = 10 -download_retry_sleep = 0.5 \ No newline at end of file +download_retry_sleep = 0.5 +sentry_url = "https://63ae106793010d836c74830fa75b300c@o264756.ingest.sentry.io/4506186624335872" \ No newline at end of file From 7da5dd2dc14d6863a2b2d4089fc7a74e0c491147 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Wed, 17 Apr 2024 12:37:39 +0200 Subject: [PATCH 30/41] default config value --- nb2workflow/nbadapter.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nb2workflow/nbadapter.py b/nb2workflow/nbadapter.py index cc9a91cf..89edf86d 100644 --- a/nb2workflow/nbadapter.py +++ b/nb2workflow/nbadapter.py @@ -256,6 +256,10 @@ def __init__(self, notebook_fn, tempdir_cache=None, config=None): self.tempdir_cache = tempdir_cache logger.debug("notebook adapter for %s", self.notebook_fn) logger.debug(self.extract_parameters()) + + if config is None: + config = dict() + self.n_download_max_tries = config.get('SERVICE.N_DOWNLOAD_MAX_TRIES', 10) self.download_retry_sleep_s = config.get('SERVICE.DOWNLOAD_RETRY_SLEEP', .5) self.max_download_size = config.get('SERVICE.MAX_DOWNLOAD_SIZE', 1e6) @@ -747,9 +751,6 @@ def base_filter(fn): good = good and re.match(pattern, os.path.basename(fn)) return good - if config is None: - config = dict() - if tests: filt = lambda fn: base_filter(fn) and "/test_" in fn else: From 904456459bedb43faf0c11c48f1e3646d076d2fb Mon Sep 17 00:00:00 2001 From: burnout87 Date: Wed, 17 Apr 2024 15:32:35 +0200 Subject: [PATCH 31/41] ontology_path config --- nb2workflow/deploy.py | 14 +++++++++++--- settings.toml | 3 ++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/nb2workflow/deploy.py b/nb2workflow/deploy.py index b8ea46bc..236b70c9 100644 --- a/nb2workflow/deploy.py +++ b/nb2workflow/deploy.py @@ -19,6 +19,7 @@ import rdflib from oda_api.ontology_helper import Ontology from nb2workflow.nbadapter import NotebookAdapter +from dynaconf import Dynaconf logger = logging.getLogger(__name__) @@ -31,6 +32,8 @@ } default_ontology_path = "https://odahub.io/ontology/ontology.ttl" +local_config = Dynaconf(settings_files=['settings.toml']) + #TODO: probably want an option to really use the dir def determine_origin(repo): @@ -609,10 +612,15 @@ def main(): parser.add_argument('--local', action="store_true", default=False) parser.add_argument('--build-engine', metavar="build_engine", default="docker") parser.add_argument('--nb2wversion', metavar="nb2wversion", default=version()) - parser.add_argument('--ontology-path', metavar="ontology_path", default=default_ontology_path) - + parser.add_argument('--ontology-path', metavar="ontology_path") + args = parser.parse_args() + ontology_path = local_config.get('default.service.ontology_path', default_ontology_path) + + if args.ontology_path is not None: + ontology_path = args.ontology_path + setup_logging() deploy(args.repository, @@ -621,7 +629,7 @@ def main(): local=args.local, build_engine=args.build_engine, nb2wversion=args.nb2wversion, - ontology_path=args.ontology_path) + ontology_path=ontology_path) if __name__ == "__main__": diff --git a/settings.toml b/settings.toml index b254dbd2..03a7b7c1 100644 --- a/settings.toml +++ b/settings.toml @@ -2,4 +2,5 @@ max_download_size = 1000000000 n_download_max_tries = 10 download_retry_sleep = 0.5 -sentry_url = "https://63ae106793010d836c74830fa75b300c@o264756.ingest.sentry.io/4506186624335872" \ No newline at end of file +sentry_url = "https://63ae106793010d836c74830fa75b300c@o264756.ingest.sentry.io/4506186624335872" +ontology_path = "https://odahub.io/ontology/ontology.ttl" \ No newline at end of file From 7adb8d403ed06fc9bbfb89a32507104b071f13a3 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Wed, 17 Apr 2024 15:34:24 +0200 Subject: [PATCH 32/41] better defaults --- nb2workflow/deploy.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/nb2workflow/deploy.py b/nb2workflow/deploy.py index 236b70c9..d4ba09b3 100644 --- a/nb2workflow/deploy.py +++ b/nb2workflow/deploy.py @@ -70,8 +70,12 @@ def build_container(git_origin, engine="docker", cleanup=False, nb2wversion=version(), - ontology_path=default_ontology_path, + ontology_path=None, **kwargs): + + if ontology_path is None: + ontology_path = local_config.get('default.service.ontology_path', default_ontology_path) + if engine == "docker": return _build_with_docker(git_origin=git_origin, local=local, @@ -570,7 +574,10 @@ def deploy(git_origin, build_timestamp=False, cleanup=False, nb2wversion=version(), - ontology_path=default_ontology_path): + ontology_path=None): + + if ontology_path is None: + ontology_path = local_config.get('default.service.ontology_path', default_ontology_path) container = build_container(git_origin, local=local, From fe31a8099205e05d0840614badfd5432e668d9d2 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Wed, 17 Apr 2024 15:37:31 +0200 Subject: [PATCH 33/41] better defaults for deploy --- nb2workflow/deploy.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/nb2workflow/deploy.py b/nb2workflow/deploy.py index d4ba09b3..556d60f4 100644 --- a/nb2workflow/deploy.py +++ b/nb2workflow/deploy.py @@ -199,7 +199,10 @@ def _build_with_kaniko(git_origin, namespace="oda-staging", cleanup=True, nb2wversion=version(), - ontology_path=default_ontology_path): + ontology_path=None): + + if ontology_path is None: + ontology_path = local_config.get('default.service.ontology_path', default_ontology_path) #secret should be created beforehand https://github.com/GoogleContainerTools/kaniko#pushing-to-docker-hub @@ -315,7 +318,11 @@ def _build_with_kaniko(git_origin, return container_metadata -def _extract_resource_requirements(local_repo_path, ontology_path=default_ontology_path): +def _extract_resource_requirements(local_repo_path, ontology_path=None): + + if ontology_path is None: + ontology_path = local_config.get('default.service.ontology_path', default_ontology_path) + ontology = Ontology(ontology_path) resources = {} @@ -344,7 +351,11 @@ def _build_with_docker(git_origin, source_from='localdir', cleanup=False, nb2wversion=version(), - ontology_path=default_ontology_path): + ontology_path=None): + + if ontology_path is None: + ontology_path = local_config.get('default.service.ontology_path', default_ontology_path) + if cleanup: logger.warning('Post-build cleanup is not implemented for docker builds') From f2be97fdbf4a3c07b94c5d48fab1c1d483b84908 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Wed, 17 Apr 2024 15:53:42 +0200 Subject: [PATCH 34/41] better way to extract default ontology path --- nb2workflow/deploy.py | 33 ++++++++++----------------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/nb2workflow/deploy.py b/nb2workflow/deploy.py index 556d60f4..1f977402 100644 --- a/nb2workflow/deploy.py +++ b/nb2workflow/deploy.py @@ -31,8 +31,8 @@ "filename_pattern": '.*', } -default_ontology_path = "https://odahub.io/ontology/ontology.ttl" local_config = Dynaconf(settings_files=['settings.toml']) +config_ontology_path = local_config.get('default.service.ontology_path', 'http://odahub.io/ontology/ontology.ttl') #TODO: probably want an option to really use the dir @@ -70,12 +70,9 @@ def build_container(git_origin, engine="docker", cleanup=False, nb2wversion=version(), - ontology_path=None, + ontology_path=config_ontology_path, **kwargs): - if ontology_path is None: - ontology_path = local_config.get('default.service.ontology_path', default_ontology_path) - if engine == "docker": return _build_with_docker(git_origin=git_origin, local=local, @@ -202,7 +199,7 @@ def _build_with_kaniko(git_origin, ontology_path=None): if ontology_path is None: - ontology_path = local_config.get('default.service.ontology_path', default_ontology_path) + ontology_path = local_config.get('default.service.ontology_path', config_ontology_path) #secret should be created beforehand https://github.com/GoogleContainerTools/kaniko#pushing-to-docker-hub @@ -318,10 +315,7 @@ def _build_with_kaniko(git_origin, return container_metadata -def _extract_resource_requirements(local_repo_path, ontology_path=None): - - if ontology_path is None: - ontology_path = local_config.get('default.service.ontology_path', default_ontology_path) +def _extract_resource_requirements(local_repo_path, ontology_path=config_ontology_path): ontology = Ontology(ontology_path) resources = {} @@ -351,10 +345,7 @@ def _build_with_docker(git_origin, source_from='localdir', cleanup=False, nb2wversion=version(), - ontology_path=None): - - if ontology_path is None: - ontology_path = local_config.get('default.service.ontology_path', default_ontology_path) + ontology_path=config_ontology_path): if cleanup: logger.warning('Post-build cleanup is not implemented for docker builds') @@ -585,11 +576,8 @@ def deploy(git_origin, build_timestamp=False, cleanup=False, nb2wversion=version(), - ontology_path=None): + ontology_path=config_ontology_path): - if ontology_path is None: - ontology_path = local_config.get('default.service.ontology_path', default_ontology_path) - container = build_container(git_origin, local=local, run_tests=run_tests, @@ -634,10 +622,9 @@ def main(): args = parser.parse_args() - ontology_path = local_config.get('default.service.ontology_path', default_ontology_path) - - if args.ontology_path is not None: - ontology_path = args.ontology_path + deploy_ontology_path = args.ontology_path + if deploy_ontology_path is None: + deploy_ontology_path = config_ontology_path setup_logging() @@ -647,7 +634,7 @@ def main(): local=args.local, build_engine=args.build_engine, nb2wversion=args.nb2wversion, - ontology_path=ontology_path) + ontology_path=deploy_ontology_path) if __name__ == "__main__": From 8cf862bcb21c9dd9e3745030edcfc7618d94b43f Mon Sep 17 00:00:00 2001 From: burnout87 Date: Wed, 17 Apr 2024 16:57:21 +0200 Subject: [PATCH 35/41] host and port in the config file --- nb2workflow/service.py | 19 +++++++++++++------ settings.toml | 2 ++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/nb2workflow/service.py b/nb2workflow/service.py index 21496ecf..ac95516d 100644 --- a/nb2workflow/service.py +++ b/nb2workflow/service.py @@ -684,9 +684,8 @@ def main(): parser = argparse.ArgumentParser(description='Process some integers.') parser.add_argument('notebook', metavar='notebook', type=str) - parser.add_argument('--host', metavar='host', - type=str, default="127.0.0.1") - parser.add_argument('--port', metavar='port', type=int, default=9191) + parser.add_argument('--host', metavar='host', type=str) + parser.add_argument('--port', metavar='port', type=int) parser.add_argument('--async-workers', metavar='N', type=int, default=3) #parser.add_argument('--tmpdir', metavar='tmpdir', type=str, default=None) parser.add_argument('--publish', metavar='upstream-url', @@ -707,6 +706,14 @@ def main(): key, value = item.split('=') app.config['SERVICE'][key] = value + service_port = app.config.get('default.service.port', 9191) + if args.port is not None: + service_port = args.port + + service_host = app.config.get('default.service.host', "127.0.0.1") + if args.host is not None: + service_host = args.host + handler = logging.StreamHandler() handler.setLevel(logging.INFO) @@ -737,13 +744,13 @@ def main(): s = args.publish_as.split(":") publish_host, publish_port = ":".join(s[:-1]), int(s[-1]) else: - publish_host, publish_port = args.host, args.port + publish_host, publish_port = service_host, service_port for nba_name, nba in app.notebook_adapters.items(): publish.publish(args.publish, nba_name, publish_host, publish_port) - # for rule in app.url_map.iter_rules(): - # logger.debug("==>> %s %s %s %s",rule,rule.endpoint,rule.__class__,rule.__dict__) + # for rule in app.url_map.iter_rules(): + # logger.debug("==>> %s %s %s %s",rule,rule.endpoint,rule.__class__,rule.__dict__) for worker_i in range(args.async_workers): async_worker = AsyncWorker('default-%i' % worker_i) diff --git a/settings.toml b/settings.toml index 03a7b7c1..b9cac5ab 100644 --- a/settings.toml +++ b/settings.toml @@ -1,4 +1,6 @@ [default.service] +host = "127.0.0.1" +port = 9191 max_download_size = 1000000000 n_download_max_tries = 10 download_retry_sleep = 0.5 From 068fa70e62971e420324fc6dbeaa756566e14334 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Fri, 19 Apr 2024 16:41:48 +0200 Subject: [PATCH 36/41] config_ontology_path --- nb2workflow/galaxy.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nb2workflow/galaxy.py b/nb2workflow/galaxy.py index ff616cfd..f682a4c9 100644 --- a/nb2workflow/galaxy.py +++ b/nb2workflow/galaxy.py @@ -15,6 +15,7 @@ import nbformat from nbconvert.exporters import ScriptExporter +from dynaconf import Dynaconf from ensureconda.api import ensureconda import subprocess as sp @@ -29,8 +30,8 @@ logger = logging.getLogger() - -default_ontology_path = 'http://odahub.io/ontology/ontology.ttl' +local_config = Dynaconf(settings_files=['settings.toml']) +config_ontology_path = local_config.get('default.service.ontology_path', 'http://odahub.io/ontology/ontology.ttl') global_req = [] @@ -539,7 +540,7 @@ def to_galaxy(input_path, citations_bibfile = None, help_file = None, available_channels = ['default', 'conda-forge'], - ontology_path = default_ontology_path, + ontology_path = config_ontology_path, test_data_baseurl = None ): @@ -690,7 +691,7 @@ def main(): tool_version = args.tool_version ontology_path = args.ontology_path if ontology_path is None: - ontology_path = default_ontology_path + ontology_path = config_ontology_path bibfile = args.citations_bibfile help_file = args.help_file test_data_baseurl = args.test_data_baseurl From ecdceba21fe4098b5b43b0c87d1331940ff4356b Mon Sep 17 00:00:00 2001 From: burnout87 Date: Tue, 23 Apr 2024 17:51:30 +0200 Subject: [PATCH 37/41] removed unused import --- nb2workflow/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nb2workflow/__init__.py b/nb2workflow/__init__.py index cc571161..cd45dbc0 100644 --- a/nb2workflow/__init__.py +++ b/nb2workflow/__init__.py @@ -1,5 +1,4 @@ import pkg_resources -import os name = "nb2workflow" From e454b75ad1634205c8117de1b0ad6595bf29e5e9 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Tue, 23 Apr 2024 20:11:42 +0200 Subject: [PATCH 38/41] arg settings path --- nb2workflow/service.py | 9 +++++++-- settings.toml | 10 ++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/nb2workflow/service.py b/nb2workflow/service.py index 0aea2470..605e3bea 100644 --- a/nb2workflow/service.py +++ b/nb2workflow/service.py @@ -96,7 +96,7 @@ def create_app(): } } - FlaskDynaconf(app, settings_files=["settings.toml"]) + FlaskDynaconf(app, settings_files=["settings.toml"], env_switcher="MERGE_ENABLED_FOR_DYNACONF") swagger = Swagger(app, template=template) app.wsgi_app = ReverseProxied(app.wsgi_app) app.json_encoder = CustomJSONEncoder @@ -699,13 +699,18 @@ def main(): parser.add_argument('--one-shot', metavar='workflow', type=str) parser.add_argument('--pattern', type=str, default=r'.*') parser.add_argument('-s', '--settings', action="append", default=None) + parser.add_argument('--settings-path', action="append", default=None) args = parser.parse_args() + if args.settings_path is not None: + print("loading settings file from ", args.settings_path[0]) + app.config.load_file(path=args.settings_path[0]) + if args.settings is not None: for item in args.settings: key, value = item.split('=') - app.config['SERVICE'][key] = value + app.config['global'][key] = value service_port = app.config.get('default.service.port', 9191) if args.port is not None: diff --git a/settings.toml b/settings.toml index b9cac5ab..b01901fe 100644 --- a/settings.toml +++ b/settings.toml @@ -1,8 +1,10 @@ -[default.service] -host = "127.0.0.1" -port = 9191 +[default.global] max_download_size = 1000000000 n_download_max_tries = 10 download_retry_sleep = 0.5 sentry_url = "https://63ae106793010d836c74830fa75b300c@o264756.ingest.sentry.io/4506186624335872" -ontology_path = "https://odahub.io/ontology/ontology.ttl" \ No newline at end of file +ontology_path = "https://odahub.io/ontology/ontology.ttl" + +[default.service] +host = "127.0.0.1" +port = 9191 From 1224d4f8685ce11d97fdc6b1e9b40851a43e4557 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Wed, 24 Apr 2024 12:18:31 +0200 Subject: [PATCH 39/41] config args --- nb2workflow/service.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nb2workflow/service.py b/nb2workflow/service.py index 605e3bea..4163b96e 100644 --- a/nb2workflow/service.py +++ b/nb2workflow/service.py @@ -698,8 +698,8 @@ def main(): parser.add_argument('--debug', action="store_true") parser.add_argument('--one-shot', metavar='workflow', type=str) parser.add_argument('--pattern', type=str, default=r'.*') - parser.add_argument('-s', '--settings', action="append", default=None) parser.add_argument('--settings-path', action="append", default=None) + parser.add_argument('-s', '--settings', nargs="*", default=[]) args = parser.parse_args() @@ -715,10 +715,12 @@ def main(): service_port = app.config.get('default.service.port', 9191) if args.port is not None: service_port = args.port + app.config['SERVICE']['port'] = service_port service_host = app.config.get('default.service.host', "127.0.0.1") if args.host is not None: service_host = args.host + app.config['service.host'] = service_host handler = logging.StreamHandler() handler.setLevel(logging.INFO) From e9f0d7275e6265c614d4c2ddf21556523a18bfff Mon Sep 17 00:00:00 2001 From: burnout87 Date: Fri, 26 Apr 2024 15:32:05 +0200 Subject: [PATCH 40/41] better way to refer to the property --- nb2workflow/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nb2workflow/service.py b/nb2workflow/service.py index 4163b96e..9ec95e4c 100644 --- a/nb2workflow/service.py +++ b/nb2workflow/service.py @@ -715,7 +715,7 @@ def main(): service_port = app.config.get('default.service.port', 9191) if args.port is not None: service_port = args.port - app.config['SERVICE']['port'] = service_port + app.config['service.port'] = service_port service_host = app.config.get('default.service.host', "127.0.0.1") if args.host is not None: From a97ec5c0e25bcec1b1bdbe768573e6f2f1db74cc Mon Sep 17 00:00:00 2001 From: burnout87 Date: Fri, 26 Apr 2024 15:32:22 +0200 Subject: [PATCH 41/41] settings path for deploy --- nb2workflow/deploy.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/nb2workflow/deploy.py b/nb2workflow/deploy.py index 37bbd7f1..bf7681b1 100644 --- a/nb2workflow/deploy.py +++ b/nb2workflow/deploy.py @@ -207,7 +207,7 @@ def _build_with_kaniko(git_origin, ontology_path=None): if ontology_path is None: - ontology_path = local_config.get('default.service.ontology_path', config_ontology_path) + ontology_path = config_ontology_path #secret should be created beforehand https://github.com/GoogleContainerTools/kaniko#pushing-to-docker-hub @@ -627,9 +627,20 @@ def main(): parser.add_argument('--build-engine', metavar="build_engine", default="docker") parser.add_argument('--nb2wversion', metavar="nb2wversion", default=version()) parser.add_argument('--ontology-path', metavar="ontology_path") + parser.add_argument('--settings-path', action="append", default=None) + parser.add_argument('-s', '--settings', nargs="*", default=[]) args = parser.parse_args() + if args.settings_path is not None: + print("loading settings file from ", args.settings_path[0]) + local_config.load_file(path=args.settings_path[0]) + + if args.settings is not None: + for item in args.settings: + key, value = item.split('=') + local_config['deploy'][key] = value + deploy_ontology_path = args.ontology_path if deploy_ontology_path is None: deploy_ontology_path = config_ontology_path