diff --git a/.github/workflows/docker_latest.yml b/.github/workflows/docker_latest.yml new file mode 100644 index 000000000000..fd564ad24589 --- /dev/null +++ b/.github/workflows/docker_latest.yml @@ -0,0 +1,54 @@ +--- +name: Build Image Latest for Docker +on: + workflow_dispatch: + inputs: + tags: + description: Comma-separated list of tags + default: latest + required: true + git-ref: + description: Git Ref + default: master + required: true + +jobs: + docker: + name: Build image and push after successfull calculation + runs-on: ubuntu-latest + + steps: + # This Checkout is necessary when using a context in docker/build-push-action + - name: Clone Repository (Latest) + uses: actions/checkout@v4 + if: github.event.inputs.git-ref == '' + - name: Clone Repository (Custom Ref) + uses: actions/checkout@v4 + if: github.event.inputs.git-ref != '' + with: + ref: ${{ github.event.inputs.git-ref }} + - name: Extract tag names + shell: bash + run: echo "##[set-output name=tags;]" | tr -d '\n'; for tag in $(echo ${{ github.event.inputs.tags }} | tr , '\n'); do echo "-t openquake/engine:$tag " | tr -d '\n'; done + id: extract_tags + - name: Build image engine with tag ${{ github.event.inputs.version }} + env: + DOCKER_USERNAME: ${{ secrets.docker_username }} + DOCKER_PASSWORD: ${{ secrets.docker_password }} + REPO_REF: ${{ github.event.inputs.git-ref }} + id: docker_engine + run: docker build --build-arg oq_branch=$REPO_REF ${{ steps.extract_tags.outputs.tags }} -f docker/Dockerfile.engine docker + - name: List Image + run: | + docker image ls + - name: Run calcs on single docker + run: | + time docker run openquake/engine:latest "oq engine --run "https://github.com/gem/oq-engine/blob/master/openquake/server/tests/data/classical.zip?raw=true"" + - name: push image engine with tags ${{ github.event.inputs.tags }} on dockerhub + env: + DOCKER_USERNAME: ${{ secrets.docker_username }} + DOCKER_PASSWORD: ${{ secrets.docker_password }} + DOCKER_TAG: ${{ github.event.inputs.version }} + run: | + docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" + docker push openquake/engine --all-tags diff --git a/.github/workflows/engine_pr_test.yml b/.github/workflows/engine_pr_test.yml index 9a3c7bb59b38..cf0c7e5201ab 100644 --- a/.github/workflows/engine_pr_test.yml +++ b/.github/workflows/engine_pr_test.yml @@ -160,6 +160,11 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - name: Actualize 'aelo' templates for email notifications + run: | + for file in openquake/server/templates/registration/*.aelo.tmpl; do + cp -- "$file" "${file%.aelo.tmpl}" + done - name: Install dependencies run: | if [[ $FROM_FORK == "true" ]]; then @@ -167,11 +172,6 @@ jobs: else python install.py devel --version=$GITHUB_HEAD_REF fi - - name: Actualize 'aelo' templates for email notifications - run: | - for file in openquake/server/templates/registration/*.aelo.tmpl; do - cp -- "$file" "${file%.aelo.tmpl}" - done - name: Server 'AELO' mode tests run: | source ~/openquake/bin/activate @@ -199,6 +199,11 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - name: Actualize 'impact' templates for email notifications + run: | + for file in openquake/server/templates/registration/*.impact.tmpl; do + cp -- "$file" "${file%.impact.tmpl}" + done - name: Install dependencies run: | if [[ $FROM_FORK == "true" ]]; then @@ -206,11 +211,6 @@ jobs: else python install.py devel --version=$GITHUB_HEAD_REF fi - - name: Actualize 'impact' templates for email notifications - run: | - for file in openquake/server/templates/registration/*.impact.tmpl; do - cp -- "$file" "${file%.impact.tmpl}" - done - name: Server 'ARISTOTLE' mode tests run: | set -x diff --git a/install.py b/install.py index 4e2ae1ef12c1..8801751a0fa4 100644 --- a/install.py +++ b/install.py @@ -210,7 +210,7 @@ class devel(user): } DEMOS = "https://artifacts.openquake.org/travis/demos-master.zip" GITBRANCH = "https://github.com/gem/oq-engine/archive/%s.zip" -URL_STANDALONE = "https://wheelhouse.openquake.org/py/standalone/latest/" +URL_STANDALONE = "https://wheelhouse.openquake.org/py/standalone/engine-3.23/" def ensure(pip=None, pyvenv=None): @@ -263,12 +263,17 @@ def get_requirements_branch(version, inst, from_fork): return version -def install_standalone(venv): +def install_or_postinstall_standalone(venv, is_install=True): """ - Install the standalone Django applications if possible + Install the standalone Django applications if possible or + run '_postinstall' command if it exists """ errors = [] - print("The standalone applications are not installed yet") + if is_install: + print("The standalone applications are not installed yet") + else: + print("Run '_postinstall' command for each standalone\n" + " Django applications, if it exists") if sys.platform == "win32": if os.path.exists("python\\python._pth.old"): pycmd = inst.VENV + "\\python.exe" @@ -276,25 +281,58 @@ def install_standalone(venv): pycmd = inst.VENV + "\\Scripts\\python.exe" else: pycmd = inst.VENV + "/bin/python3" - for app in [ - "oq-platform-standalone", - "oq-platform-ipt", - "oq-platform-taxonomy", - ]: - try: - print("Applications " + app + " are not installed yet \n") - subprocess.check_call( - [pycmd, "-m", "pip", "install", "--find-links", URL_STANDALONE, - app] - ) - except Exception as exc: - # for instance is somebody removed a wheel from the wheelhouse - errors.append("%s: could not install %s" % (exc, app)) + STANDALONE_APP_INFO = [ + {"pkg": "oq-platform-standalone", "name": None}, + {"pkg": "oq-platform-ipt", "name": "openquakeplatform_ipt"}, + {"pkg": "oq-platform-taxonomy", "name": "openquakeplatform_taxonomy"}, + {"pkg": "django-gem-taxonomy", "name": "django_gem_taxonomy"}, + ] + + if is_install: + for app in STANDALONE_APP_INFO: + try: + print("Applications " + app['pkg'] + " are not installed yet \n") + + subprocess.check_call( + [pycmd, "-m", "pip", "install", "--find-links", URL_STANDALONE, + app['pkg']] + ) + except Exception as exc: + # for instance is somebody removed a wheel from the wheelhouse + errors.append("%s: could not install %s" % (exc, app['pkg'])) + else: + for app in STANDALONE_APP_INFO: + if not app['name']: + continue + + try: + if sys.platform == "win32": + django_admin = ['Scripts', 'django-admin.exe'] + else: + django_admin = ["bin", "django-admin"] + + django_env = os.environ.copy() + django_env["DJANGO_SETTINGS_MODULE"] = "openquake.server.settings" + + subprocess.check_call( + [os.path.join(inst.VENV, *django_admin), + "openquake_engine_postinstall", app['name']], + env=django_env) + except Exception as exc: + # for instance is somebody removed a wheel from the wheelhouse + errors.append("%s: error during %s postinstall command execution" % (exc, app['name'])) + return errors -def before_checks(inst, venv, port, remove, usage): +def install_standalone(venv): + return install_or_postinstall_standalone(venv, is_install=True) + +def postinstall_standalone(venv): + return install_or_postinstall_standalone(venv, is_install=False) + +def before_checks(inst, venv, port, remove, novenv, usage): """ Checks to perform before the installation """ @@ -303,6 +341,10 @@ def before_checks(inst, venv, port, remove, usage): if port: inst.DBPORT = int(port) + if novenv: + inst.VENV = os.path.join(os.getenv('LocalAppData'), 'Programs', + 'OpenQuake Engine', 'python3') + # check platform if (inst is server and sys.platform != "linux") or ( inst is devel_server and sys.platform != "linux" @@ -386,7 +428,7 @@ def fix_version(commit, venv): f.write("".join(lines)) -def install(inst, version, from_fork): +def install(inst, version, from_fork, novenv, noupgrade): """ Install the engine in one of the three possible modes """ @@ -406,34 +448,38 @@ def install(inst, version, from_fork): if inst is server or inst is devel_server: subprocess.check_call(["chown", "openquake", inst.OQDATA]) - # recreate the openquake venv - ensure(pyvenv=inst.VENV) - print("Created %s" % inst.VENV) + if not novenv: + # recreate the openquake venv + ensure(pyvenv=inst.VENV) + print("Created %s" % inst.VENV) - if sys.platform == "win32": - if os.path.exists("python\\python._pth.old"): - pycmd = inst.VENV + "\\python.exe" + if sys.platform == "win32": + if os.path.exists("python\\python._pth.old"): + pycmd = inst.VENV + "\\python.exe" + else: + pycmd = inst.VENV + "\\Scripts\\python.exe" else: - pycmd = inst.VENV + "\\Scripts\\python.exe" + pycmd = inst.VENV + "/bin/python3" else: - pycmd = inst.VENV + "/bin/python3" - + pycmd = os.path.join(inst.VENV, 'python.exe') # upgrade pip and before check that it is installed in venv if sys.platform != "win32": ensure(pip=pycmd) - subprocess.check_call( - [pycmd, "-m", "pip", "install", "--upgrade", "pip", "wheel"] - ) + subprocess.check_call([pycmd, "-m", "pip", "install"] + ([ + ] if noupgrade else ["--upgrade"]) + [ + "pip", "wheel"]) else: if os.path.exists("python\\python._pth.old"): - subprocess.check_call( - [pycmd, "-m", "pip", "install", "--upgrade", "pip", "wheel", - "urllib3"]) + subprocess.check_call([pycmd, "-m", "pip", "install"] + ([ + ] if noupgrade else ["--upgrade"]) + [ + "pip", "wheel", "urllib3"]) else: - subprocess.check_call([pycmd, "-m", "ensurepip", "--upgrade"]) - subprocess.check_call( - [pycmd, "-m", "pip", "install", "--upgrade", "pip", "wheel", - "urllib3"]) + subprocess.check_call([pycmd, "-m", "ensurepip"] + ([ + ] if noupgrade else ["--upgrade"])) + + subprocess.check_call([pycmd, "-m", "pip", "install"] + ([ + ] if noupgrade else ["--upgrade"]) + [ + "pip", "wheel", "urllib3"]) # install the requirements branch = get_requirements_branch(version, inst, from_fork) @@ -465,18 +511,20 @@ def install(inst, version, from_fork): subprocess.check_call([pycmd, "-m", "pip", "install", "-e", CDIR]) elif version is None: # install the stable version subprocess.check_call( - [pycmd, "-m", "pip", "install", "--upgrade", "openquake.engine"] + [pycmd, "-m", "pip", "install"] + ([ + ] if noupgrade else ["--upgrade"]) + ["openquake.engine"] ) elif re.match(r"\d+(\.\d+)+", version): # install an official version subprocess.check_call( - [pycmd, "-m", "pip", "install", "--upgrade", - "openquake.engine==" + version] + [pycmd, "-m", "pip", "install"] + ([ + ] if noupgrade else ["--upgrade"]) + ["openquake.engine==" + version] ) else: # install a branch from github (only for user or server) commit = latest_commit(version) print("Installing commit", commit) subprocess.check_call( - [pycmd, "-m", "pip", "install", "--upgrade", GITBRANCH % commit] + [pycmd, "-m", "pip", "install"] + ([ + ] if noupgrade else ["--upgrade"]) + [GITBRANCH % commit] ) fix_version(commit, inst.VENV) @@ -506,6 +554,8 @@ def install(inst, version, from_fork): if inst in (user, devel): # create/upgrade the db in the default location subprocess.run([oqreal, "engine", "--upgrade-db"]) + errors += postinstall_standalone(inst.VENV) + if ( inst is server and not os.path.exists(inst.OQ) @@ -610,6 +660,10 @@ def remove(inst): help="the kind of installation you want", ) parser.add_argument("--venv", help="venv directory") + parser.add_argument("--novenv", action="store_true", + help="keep the current python environment") + parser.add_argument("--noupgrade", action="store_true", + help="not use '--upgrade' in pip install calls") parser.add_argument("--remove", action="store_true", help="disinstall the engine") parser.add_argument("--version", help="version to install (default stable)") @@ -623,12 +677,12 @@ def remove(inst): args = parser.parse_args() if args.inst: inst = globals()[args.inst] - before_checks(inst, args.venv, args.dbport, args.remove, + before_checks(inst, args.venv, args.dbport, args.remove, args.novenv, parser.format_usage()) if args.remove: remove(inst) else: - errors = install(inst, args.version, args.from_fork) + errors = install(inst, args.version, args.from_fork, args.novenv, args.noupgrade) if errors: # NB: even if one of the tools is missing, the engine will work sys.exit('\n'.join(errors)) diff --git a/openquake/__init__.py b/openquake/__init__.py deleted file mode 100644 index 9600bd23ce85..000000000000 --- a/openquake/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: tabstop=4 shiftwidth=4 softtabstop=4 -# -# Copyright (C) 2010-2025 GEM Foundation -# -# OpenQuake is free software: you can redistribute it and/or modify it -# under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# OpenQuake is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with OpenQuake. If not, see . - -__import__('pkg_resources').declare_namespace(__name__) diff --git a/openquake/server/management/commands/openquake_engine_postinstall.py b/openquake/server/management/commands/openquake_engine_postinstall.py new file mode 100644 index 000000000000..f09f57c7c4da --- /dev/null +++ b/openquake/server/management/commands/openquake_engine_postinstall.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# oq-geoviewer +# Copyright (C) 2018-2019 GEM Foundation +# +# oq-geoviewer is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# oq-geoviewer is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# import subprocess +from django.apps import apps as django_apps +from django.core.management import call_command, get_commands +import sys +from django.core.management.base import BaseCommand + +class Command(BaseCommand): + help = ("Command that run a '_postinstall' command if it exists") + + def add_arguments(self, parser): + parser.add_argument('django_app', + help='django application name') + + def handle(self, *args, **options): + found = False + for app in django_apps.get_app_configs(): + label = app.label + if options['django_app'] == label: + found = True + break + + if not found: + self.stdout.write( + self.style.ERROR( + "No django app '%s' found." % (options['django_app'],)) + ) + sys.exit(1) + + postinstall_cmd = options['django_app'] + '_postinstall' + django_cmds = get_commands() + if postinstall_cmd not in django_cmds: + self.stdout.write( + self.style.WARNING( + "No 'postinst' action needed for app %s, skipped." % (options['django_app'],)) + ) + sys.exit(0) + + call_command(postinstall_cmd) diff --git a/openquake/server/settings.py b/openquake/server/settings.py index 0b98204df711..3f1ed63a022c 100644 --- a/openquake/server/settings.py +++ b/openquake/server/settings.py @@ -26,8 +26,6 @@ from openquake.baselib import config from openquake.commonlib import datastore -# optionally overrided in local_settings.py -STANDALONE_APP_NAME_MAP = {} try: from openquakeplatform.settings import STANDALONE, STANDALONE_APPS except ImportError: @@ -213,6 +211,14 @@ APPLICATION_MODE = 'PUBLIC' +# Definition of Django applications +STANDALONE_APP_NAME_MAP = { + 'openquakeplatform_ipt': 'ipt', + 'django_gem_taxonomy': 'taxonomy', + } +if APPLICATION_MODE != 'TOOLS_ONLY': + STANDALONE_APP_NAME_MAP['openquakeplatform_taxonomy'] = 'glossary' + ARISTOTLE_DEFAULT_USGS_ID = 'us7000n7n8' # loadable and convertible rupture # ARISTOTLE_DEFAULT_USGS_ID = 'us6000jllz' # loadable but with conversion err diff --git a/pytest.ini b/pytest.ini index 11455d4fd9b5..dfdd5512968a 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,4 +2,5 @@ markers = slow: a slow test addopts = --tb short +consider_namespace_packages = true DJANGO_SETTINGS_MODULE = openquake.server.settings diff --git a/setup.py b/setup.py index b22ca2061019..13d3d23a6185 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,8 @@ import os import re import sys -from setuptools import setup, find_packages +from setuptools import setup, find_namespace_packages + if sys.version_info < (3, 9): sys.exit('Sorry, Python < 3.9 is not supported') @@ -112,11 +113,16 @@ def get_readme(): 'Environment :: Console', 'Environment :: Web Environment', ], - packages=find_packages(exclude=["qa_tests", "qa_tests.*", - "tools", - "*.*.tests", "*.*.tests.*", - "openquake.engine.bin", - "openquake.engine.bin.*"]), + packages=find_namespace_packages(include=[ + "openquake.*", + ], + exclude=[ + "qa_tests", "qa_tests.*", + "tools", + "*.*.tests", "*.*.tests.*", + "openquake.engine.bin", + "openquake.engine.bin.*", + ]), py_modules=PY_MODULES, include_package_data=True, package_data={"openquake.engine": [