diff --git a/.github/workflows/webui_rh_test.yml b/.github/workflows/webui_rh_test.yml index 75c0fb5aebfe..45719fa4f605 100644 --- a/.github/workflows/webui_rh_test.yml +++ b/.github/workflows/webui_rh_test.yml @@ -66,25 +66,32 @@ jobs: eval '$mypython -m pip install -e . ' ## Standalone apps echo "Downloading standalone apps" - for app in oq-platform-standalone oq-platform-ipt oq-platform-taxonomy; do - # do not fail on exit code 2 if the branch - # does not exist in the app repository - set +e - git ls-remote --exit-code --heads https://github.com/gem/${app}.git $BRANCH >/dev/null 2>&1 - EXIT_CODE=$? - # set again to fail on exit code not 0 - set -e - if [[ $EXIT_CODE == '0' ]]; then - echo "Git branch '$BRANCH' exists in the remote repository" - TOOLS_BRANCH=$BRANCH - elif [[ $EXIT_CODE == '2' ]]; then - echo "Git branch '$BRANCH' does not exist in the remote repository" - TOOLS_BRANCH=master - fi - echo "We need to use the branch $TOOLS_BRANCH for the standalone apps" - git clone -b ${TOOLS_BRANCH} --depth=1 https://github.com/gem/${app}.git - eval '$mypython -m pip install -e ./${app}' + for app in oq-platform-standalone django-gem-taxonomy oq-platform-ipt oq-platform-taxonomy; do + for branch in "$BRANCH" "master" "main"; do + TOOLS_BRANCH="$branch" + # do not fail on exit code 2 if the branch + # does not exist in the app repository + set +e + git ls-remote --exit-code --heads https://github.com/gem/${app}.git $TOOLS_BRANCH >/dev/null 2>&1 + EXIT_CODE=$? + # set again to fail on exit code not 0 + set -e + if [[ $EXIT_CODE -eq 0 ]]; then + echo "Git branch '$BRANCH' exists in the remote repository" + break + fi + done + if [[ $EXIT_CODE -ne 0 ]]; then + break + fi + echo "We need to use the branch $TOOLS_BRANCH for the standalone apps" + git clone -b ${TOOLS_BRANCH} --depth=1 https://github.com/gem/${app}.git + eval '$mypython -m pip install -e ./${app}' done + if [ $EXIT_CODE -ne 0 ]; then + echo "No '$branch', nor 'master' or 'main' branch found for '$app' django app; failed" + exit 1 + fi deactivate - name: Actualize 'default' templates for email notifications run: | diff --git a/install.py b/install.py index 747e1fe8280d..1aeb0221693e 100644 --- a/install.py +++ b/install.py @@ -208,7 +208,9 @@ class devel(user): "win32": ("win64",), } GITBRANCH = "https://github.com/gem/oq-engine/archive/%s.zip" -URL_STANDALONE = "https://wheelhouse.openquake.org/py/standalone/latest/" +# FIXME just for devel test +# URL_STANDALONE = "https://wheelhouse.openquake.org/py/standalone/latest/" +URL_STANDALONE = "https://wheelhouse.openquake.org/py/standalone/post-inst/" def ensure(pip=None, pyvenv=None): @@ -260,12 +262,17 @@ def get_requirements_branch(version, inst, from_fork): return version -def install_standalone(venv): +def install_or_postinstall_standalone(venv, is_install=True): """ - Install the standalone Django applications if possible + Install the standalone Django applications if possible or + run '_postinstall' command if it exists """ errors = [] - print("The standalone applications are not installed yet") + if is_install: + print("The standalone applications are not installed yet") + else: + print("Run '_postinstall' command for each standalone\n" + " Django applications, if it exists") if sys.platform == "win32": if os.path.exists("python\\python._pth.old"): pycmd = inst.VENV + "\\python.exe" @@ -273,24 +280,57 @@ def install_standalone(venv): pycmd = inst.VENV + "\\Scripts\\python.exe" else: pycmd = inst.VENV + "/bin/python3" - for app in [ - "oq-platform-standalone", - "oq-platform-ipt", - "oq-platform-taxonomy", - ]: - try: - print("Applications " + app + " are not installed yet \n") - subprocess.check_call( - [pycmd, "-m", "pip", "install", "--find-links", URL_STANDALONE, - app] - ) - except Exception as exc: - # for instance is somebody removed a wheel from the wheelhouse - errors.append("%s: could not install %s" % (exc, app)) + STANDALONE_APP_INFO = [ + {"pkg": "oq-platform-standalone", "name": None}, + {"pkg": "oq-platform-ipt", "name": "openquakeplatform_ipt"}, + {"pkg": "oq-platform-taxonomy", "name": "openquakeplatform_taxonomy"}, + {"pkg": "django-gem-taxonomy", "name": "django_gem_taxonomy"}, + ] + + if is_install: + for app in STANDALONE_APP_INFO: + try: + print("Applications " + app['pkg'] + " are not installed yet \n") + + subprocess.check_call( + [pycmd, "-m", "pip", "install", "--find-links", URL_STANDALONE, + app['pkg']] + ) + except Exception as exc: + # for instance is somebody removed a wheel from the wheelhouse + errors.append("%s: could not install %s" % (exc, app['pkg'])) + else: + for app in STANDALONE_APP_INFO: + if not app['name']: + continue + + try: + if sys.platform == "win32": + django_admin = ['Scripts', 'django-admin.exe'] + else: + django_admin = ["bin", "django-admin"] + + django_env = os.environ.copy() + django_env["DJANGO_SETTINGS_MODULE"] = "openquake.server.settings" + + subprocess.check_call( + [os.path.join(inst.VENV, *django_admin), + "openquake_engine_postinstall", app['name']], + env=django_env) + except Exception as exc: + # for instance is somebody removed a wheel from the wheelhouse + errors.append("%s: error during %s postinstall command execution" % (exc, app['name'])) + return errors +def install_standalone(venv): + return install_or_postinstall_standalone(venv, is_install=True) + +def postinstall_standalone(venv): + return install_or_postinstall_standalone(venv, is_install=False) + def before_checks(inst, args, usage): """ Checks to perform before the installation @@ -300,6 +340,10 @@ def before_checks(inst, args, usage): if args.dbport: inst.DBPORT = int(args.dbport) + if args.novenv: + inst.VENV = os.path.join(os.getenv('LocalAppData'), 'Programs', + 'OpenQuake Engine', 'python3') + # check platform if (inst is server and sys.platform != "linux") or ( inst is devel_server and sys.platform != "linux" @@ -416,7 +460,7 @@ def normalize_version(version): return f"=={version}" -def install(inst, version, from_fork): +def install(inst, version, from_fork, novenv, noupgrade): """ Install the engine in one of the three possible modes """ @@ -436,34 +480,38 @@ def install(inst, version, from_fork): if inst is server or inst is devel_server: subprocess.check_call(["chown", "openquake", inst.OQDATA]) - # recreate the openquake venv - ensure(pyvenv=inst.VENV) - print("Created %s" % inst.VENV) + if not novenv: + # recreate the openquake venv + ensure(pyvenv=inst.VENV) + print("Created %s" % inst.VENV) - if sys.platform == "win32": - if os.path.exists("python\\python._pth.old"): - pycmd = inst.VENV + "\\python.exe" + if sys.platform == "win32": + if os.path.exists("python\\python._pth.old"): + pycmd = inst.VENV + "\\python.exe" + else: + pycmd = inst.VENV + "\\Scripts\\python.exe" else: - pycmd = inst.VENV + "\\Scripts\\python.exe" + pycmd = inst.VENV + "/bin/python3" else: - pycmd = inst.VENV + "/bin/python3" + pycmd = os.path.join(inst.VENV, 'python.exe') # upgrade pip and before check that it is installed in venv if sys.platform != "win32": ensure(pip=pycmd) - subprocess.check_call( - [pycmd, "-m", "pip", "install", "--upgrade", "pip", "wheel"] - ) + subprocess.check_call([pycmd, "-m", "pip", "install"] + ([ + ] if noupgrade else ["--upgrade"]) + [ + "pip", "wheel"]) else: if os.path.exists("python\\python._pth.old"): - subprocess.check_call( - [pycmd, "-m", "pip", "install", "--upgrade", "pip", "wheel", - "urllib3"]) + subprocess.check_call([pycmd, "-m", "pip", "install"] + ([ + ] if noupgrade else ["--upgrade"]) + [ + "pip", "wheel", "urllib3"]) else: - subprocess.check_call([pycmd, "-m", "ensurepip", "--upgrade"]) - subprocess.check_call( - [pycmd, "-m", "pip", "install", "--upgrade", "pip", "wheel", - "urllib3"]) + subprocess.check_call([pycmd, "-m", "ensurepip"] + ([ + ] if noupgrade else ["--upgrade"])) + subprocess.check_call([pycmd, "-m", "pip", "install"] + ([ + ] if noupgrade else ["--upgrade"]) + [ + "pip", "wheel", "urllib3"]) # install the requirements branch = get_requirements_branch(version, inst, from_fork) @@ -494,12 +542,14 @@ def install(inst, version, from_fork): subprocess.check_call([pycmd, "-m", "pip", "install", "-e", CDIR]) elif version is None: # install the stable version subprocess.check_call( - [pycmd, "-m", "pip", "install", "--upgrade", "openquake.engine"] + [pycmd, "-m", "pip", "install"] + ([ + ] if noupgrade else ["--upgrade"]) + ["openquake.engine"] ) elif re.match(r"\d+(\.\d+)+", version): # install an official version subprocess.check_call( - [pycmd, "-m", "pip", "install", "--upgrade", - f"openquake.engine{normalize_version(version)}"] + [pycmd, "-m", "pip", "install"] + ([] if noupgrade + else ["--upgrade"]) + + [f"openquake.engine{normalize_version(version)}"] ) else: # install a branch from github (only for user or server) commit = latest_commit(version) @@ -509,8 +559,9 @@ def install(inst, version, from_fork): custom_env["TMPDIR"] = tmp # Linux/macOS custom_env["TEMP"] = tmp # Windows subprocess.check_call( - [pycmd, "-m", "pip", "install", - "--upgrade", GITBRANCH % commit, "--no-clean"], + [pycmd, "-m", "pip", "install"] + ( + [] if noupgrade else ["--upgrade"]) + [GITBRANCH % commit, + "--no-clean"], env=custom_env) fix_version(commit, inst.VENV) @@ -541,6 +592,8 @@ def install(inst, version, from_fork): if inst in (user, devel): # create/upgrade the db in the default location subprocess.run([oqreal, "engine", "--upgrade-db"]) + errors += postinstall_standalone(inst.VENV) + if ( inst is server and not os.path.exists(inst.OQ) @@ -634,6 +687,10 @@ def remove(inst): help="the kind of installation you want", ) parser.add_argument("--venv", help="venv directory") + parser.add_argument("--novenv", action="store_true", + help="keep the current python environment") + parser.add_argument("--noupgrade", action="store_true", + help="not use '--upgrade' in pip install calls") parser.add_argument("--remove", action="store_true", help="disinstall the engine") parser.add_argument("--version", help="version to install (default stable)") @@ -651,7 +708,7 @@ def remove(inst): if args.remove: remove(inst) else: - errors = install(inst, args.version, args.from_fork) + errors = install(inst, args.version, args.from_fork, args.novenv, args.noupgrade) if errors: # NB: even if one of the tools is missing, the engine will work sys.exit('\n'.join(errors)) diff --git a/openquake/server/management/commands/openquake_engine_postinstall.py b/openquake/server/management/commands/openquake_engine_postinstall.py new file mode 100644 index 000000000000..f09f57c7c4da --- /dev/null +++ b/openquake/server/management/commands/openquake_engine_postinstall.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# oq-geoviewer +# Copyright (C) 2018-2019 GEM Foundation +# +# oq-geoviewer is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# oq-geoviewer is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# import subprocess +from django.apps import apps as django_apps +from django.core.management import call_command, get_commands +import sys +from django.core.management.base import BaseCommand + +class Command(BaseCommand): + help = ("Command that run a '_postinstall' command if it exists") + + def add_arguments(self, parser): + parser.add_argument('django_app', + help='django application name') + + def handle(self, *args, **options): + found = False + for app in django_apps.get_app_configs(): + label = app.label + if options['django_app'] == label: + found = True + break + + if not found: + self.stdout.write( + self.style.ERROR( + "No django app '%s' found." % (options['django_app'],)) + ) + sys.exit(1) + + postinstall_cmd = options['django_app'] + '_postinstall' + django_cmds = get_commands() + if postinstall_cmd not in django_cmds: + self.stdout.write( + self.style.WARNING( + "No 'postinst' action needed for app %s, skipped." % (options['django_app'],)) + ) + sys.exit(0) + + call_command(postinstall_cmd) diff --git a/openquake/server/settings.py b/openquake/server/settings.py index 35b50ea0a6d3..69ebab350fcd 100644 --- a/openquake/server/settings.py +++ b/openquake/server/settings.py @@ -230,6 +230,14 @@ # IMPACT_DEFAULT_USGS_ID = 'us7000n7n8' # loadable and convertible rupture # IMPACT_DEFAULT_USGS_ID = 'us6000jllz' # loadable but with conversion err +# Definition of Django applications +STANDALONE_APP_NAME_MAP = { + 'openquakeplatform_ipt': 'ipt', + 'django_gem_taxonomy': 'taxonomy', + } +if APPLICATION_MODE != 'TOOLS_ONLY': + STANDALONE_APP_NAME_MAP['openquakeplatform_taxonomy'] = 'glossary' + EXTERNAL_TOOLS = os.environ.get('EXTERNAL_TOOLS', False) == 'True' # If False, a warning is displayed in case a newer version of the engine has