diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f70b4f14..1e3548b0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -154,7 +154,7 @@ jobs: - uses: 'google-github-actions/auth@v3' with: service_account: 'github-actions-sa@sdss-sdss-invest-compute.iam.gserviceaccount.com' - workload_identity_provider: 'projects/616945215462/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions' + workload_identity_provider: 'projects/616945215462/locations/global/workloadIdentityPools/github-actions-pool-910c7e51/providers/github-actions' - name: Install package run: cd invest_processes && pip install . diff --git a/invest_processes/pyproject.toml b/invest_processes/pyproject.toml index f58dfa6e..d0a2bc4f 100644 --- a/invest_processes/pyproject.toml +++ b/invest_processes/pyproject.toml @@ -10,7 +10,6 @@ maintainers = [ dynamic = ["version"] # the version is provided dynamically by setuptools_scm dependencies = [ "flask", - "natcap.invest", "pygeoapi>=0.23.4", "google-cloud-storage" ] diff --git a/invest_processes/src/invest_processes/execute.py b/invest_processes/src/invest_processes/execute.py index de287543..332934e8 100644 --- a/invest_processes/src/invest_processes/execute.py +++ b/invest_processes/src/invest_processes/execute.py @@ -2,7 +2,6 @@ from pathlib import Path import textwrap -from invest_processes.utils import download_and_extract_datastack from pygeoapi.process.base import BaseProcessor, ProcessorExecuteError LOGGER = logging.getLogger(__name__) @@ -19,9 +18,12 @@ 'jobControlOptions': ['async-execute'], 'keywords': ['invest'], 'inputs': { - 'datastack_path': { - 'title': 'Datastack path', - 'description': 'The path to the datastack JSON file to execute', + 'datastack_url': { + 'title': 'Datastack URL', + 'description': ( + 'The URL to a downloadable tar.gz archive of the datastack to run. ' + 'Must be formatted as an InVEST datastack containing a JSON parameters ' + 'file called parameters.invest.json, as well as all input data files.'), 'schema': { 'type': 'string' }, @@ -41,7 +43,7 @@ }, 'example': { 'inputs': { - 'datastack_path': '/Users/emily/invest/data/Carbon/carbon_willamette.invs.json', + 'datastack_url': 'https://raw.githubusercontent.com/natcap/invest-compute/refs/heads/main/tests/test_data/invest_carbon_datastack.tgz' } } } @@ -76,11 +78,15 @@ def create_slurm_script(self, datastack_url, workspace_dir): json_path = f'{workspace_dir}/datastack/parameters.invest.json' return textwrap.dedent(f"""\ #!/bin/sh + #SBATCH --exclusive curl -o datastack.tgz "{datastack_url}" mkdir {workspace_dir}/datastack tar -xzvf datastack.tgz -C {workspace_dir}/datastack rm datastack.tgz + + eval "$(~/bin/micromamba shell hook -s posix)" + micromamba activate invest_env MODEL_ID=$(python -c "from natcap.invest import datastack; print(datastack.extract_parameter_set('{json_path}').model_id)") invest --debug --taskgraph-log-level=DEBUG run \ --datastack {json_path} \ diff --git a/invest_processes/src/invest_processes/slurm_manager.py b/invest_processes/src/invest_processes/slurm_manager.py index 394adfda..4f2c4b15 100644 --- a/invest_processes/src/invest_processes/slurm_manager.py +++ b/invest_processes/src/invest_processes/slurm_manager.py @@ -714,7 +714,7 @@ def submit_slurm_job(self, processor, data_dict): LOGGER.info(f'stdout from sbatch: {result.stdout}') except subprocess.CalledProcessError as e: - raise RuntimeError('Error when submitting slurm job') from e + raise RuntimeError(f'Error when submitting slurm job: {e.stderr}') from e job_id = result.stdout.strip() LOGGER.info(f"Job submitted successfully with ID: {job_id}") diff --git a/invest_processes/src/invest_processes/utils.py b/invest_processes/src/invest_processes/utils.py deleted file mode 100644 index cfe60a10..00000000 --- a/invest_processes/src/invest_processes/utils.py +++ /dev/null @@ -1,52 +0,0 @@ -import os -import tarfile -import tempfile - -from natcap.invest import datastack -from pygeoapi.process.base import ProcessorExecuteError -import requests - - -def download_and_extract_datastack(datastack_url, extracted_datastack_dir): - """Download and extract a datastack tgz archive to a given local directory. - - Args: - datastack_url (str): URL to download the datastack archive from - extracted_datastack_dir (str): local directory path to extract to - - Returns: - tuple of extracted json datastack path and model id - """ - # Download the datastack from the given URL and - response = requests.get(datastack_url, stream=True) - if response.status_code != 200: - raise ProcessorExecuteError( - "Failed to download datastack file. Request returned " + - {response.status_code}) - - with tempfile.TemporaryDirectory() as temp_dir: - # save the datastack archive to a local temp file - tgz_path = os.path.join(temp_dir, 'datastack.tgz') - with open(tgz_path, 'wb') as tgz: - # chunk_size defaults to 1 byte which is unnecessarily small - for chunk in response.iter_content(chunk_size=1024): - tgz.write(chunk) - - # extract the TGZ archive to a local directory - try: - with tarfile.open(tgz_path, 'r:gz') as tgz: - tgz.extractall(path=extracted_datastack_dir, filter='data') - except Exception as err: - raise ProcessorExecuteError( - 1, f'Failed to extract datastack archive:\n{str(err)}') - - # Parse the extracted datastack JSON. Datastack archives created in the - # workbench should have the JSON file named parameters.invest.json. - json_path = os.path.join(extracted_datastack_dir, 'parameters.invest.json') - try: - model_id = datastack.extract_parameter_set(json_path).model_id - except Exception as error: - raise ProcessorExecuteError( - 1, f'Error when parsing JSON datastack:\n{str(error)}') - - return json_path, model_id diff --git a/invest_processes/src/invest_processes/validate.py b/invest_processes/src/invest_processes/validate.py index 6dcf5987..92f491bc 100644 --- a/invest_processes/src/invest_processes/validate.py +++ b/invest_processes/src/invest_processes/validate.py @@ -4,7 +4,6 @@ import textwrap import time -from invest_processes.utils import download_and_extract_datastack from pygeoapi.process.base import BaseProcessor, ProcessorExecuteError LOGGER = logging.getLogger(__name__) @@ -21,9 +20,12 @@ 'jobControlOptions': ['async-execute', 'sync-execute'], 'keywords': ['invest'], 'inputs': { - 'datastack_path': { - 'title': 'Datastack path', - 'description': 'The path to the datastack JSON file to validate', + 'datastack_url': { + 'title': 'Datastack URL', + 'description': ( + 'The URL to a downloadable tar.gz archive of the datastack to run. ' + 'Must be formatted as an InVEST datastack containing a JSON parameters ' + 'file called parameters.invest.json, as well as all input data files.'), 'schema': { 'type': 'string' }, @@ -57,7 +59,7 @@ }, 'example': { 'inputs': { - 'datastack_path': '/Users/emily/invest/data/Carbon/carbon_willamette.invs.json', + 'datastack_url': 'https://raw.githubusercontent.com/natcap/invest-compute/refs/heads/main/tests/test_data/invest_carbon_datastack.tgz' } } } diff --git a/invest_processes/startup_script.sh b/invest_processes/startup_script.sh new file mode 100644 index 00000000..adc0224f --- /dev/null +++ b/invest_processes/startup_script.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +scl enable gcc-toolset-12 bash + +# install micromamba +curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj ~/bin/micromamba +eval "$(~/bin/micromamba shell hook -s posix)" +micromamba --version + +# create and activate the environment +micromamba create -y -n invest_env python=3.13 gdal +micromamba activate invest_env +pip install setuptools setuptools_scm build cython babel +pip install --no-build-isolation git+https://github.com/natcap/pygeoprocessing.git +pip install --no-build-isolation git+https://github.com/emlys/invest.git@bugfix/2572 +invest --version + +# activate the environment in future shells +echo 'echo "activating micromamba env"' >> /etc/bashrc +echo 'eval "$(~/bin/micromamba shell hook -s posix)" && micromamba activate invest_env' >> ~/.bashrc diff --git a/slurm_cluster_config/hpc-slurm/main.tf b/slurm_cluster_config/hpc-slurm/main.tf index b7b86318..a8101d7b 100644 --- a/slurm_cluster_config/hpc-slurm/main.tf +++ b/slurm_cluster_config/hpc-slurm/main.tf @@ -58,18 +58,20 @@ module "homefs" { region = var.region reserved_ip_range = module.private_service_access.reserved_ip_range zone = var.zone + size_gb = 2560 } module "debug_nodeset" { source = "github.com/GoogleCloudPlatform/cluster-toolkit//community/modules/compute/schedmd-slurm-gcp-v6-nodeset?ref=v1.90.0" allow_automatic_updates = false labels = var.labels - machine_type = "c2-standard-8" + machine_type = "c2-standard-4" disk_type = "pd-ssd" name = "debug_nodeset" node_count_dynamic_max = 4 project_id = var.project_id region = var.region + startup_script = "/home/bin/startup_script.sh" subnetwork_self_link = module.network.subnetwork_self_link zone = var.zone } @@ -87,12 +89,12 @@ module "compute_nodeset" { allow_automatic_updates = false bandwidth_tier = "gvnic_enabled" labels = var.labels - machine_type = "c2-standard-8" + machine_type = "c2-standard-4" name = "compute_nodeset" node_count_dynamic_max = 20 project_id = var.project_id region = var.region - startup_script = "startup_script.sh" + startup_script = "/home/bin/startup_script.sh" subnetwork_self_link = module.network.subnetwork_self_link zone = var.zone } @@ -128,7 +130,7 @@ module "slurm_login" { source = "github.com/GoogleCloudPlatform/cluster-toolkit//community/modules/scheduler/schedmd-slurm-gcp-v6-login?ref=v1.90.0" enable_login_public_ips = true labels = var.labels - machine_type = "n2-standard-4" + machine_type = "n2-standard-2" name_prefix = "slurm_login" project_id = var.project_id region = var.region @@ -143,6 +145,7 @@ module "slurm_controller" { enable_controller_public_ips = true labels = var.labels login_nodes = flatten([module.slurm_login.login_nodes]) + machine_type = "n2-standard-2" network_storage = flatten([module.homefs.network_storage]) nodeset = flatten([module.h3_partition.nodeset, flatten([module.compute_partition.nodeset, flatten([module.debug_partition.nodeset])])]) nodeset_dyn = flatten([module.h3_partition.nodeset_dyn, flatten([module.compute_partition.nodeset_dyn, flatten([module.debug_partition.nodeset_dyn])])]) diff --git a/slurm_cluster_config/login_node_playbook.yml b/slurm_cluster_config/login_node_playbook.yml index 111f1e63..ce569c8d 100644 --- a/slurm_cluster_config/login_node_playbook.yml +++ b/slurm_cluster_config/login_node_playbook.yml @@ -42,6 +42,13 @@ - "--exclude=slurm_cluster_config" - "--exclude=tests" + - name: Move startup script to shared location + become: true + ansible.builtin.shell: | + mkdir /home/bin + mv /home/esoth_stanford_edu/invest-compute/invest_processes/startup_script.sh /home/bin + chmod +x /home/bin/startup_script.sh + - name: Install and launch server args: chdir: invest-compute/invest_processes @@ -49,13 +56,13 @@ ansible.builtin.shell: | # install micromamba rm -f bin/micromamba - curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba - eval "$(./bin/micromamba shell hook -s posix)" + curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj ~/bin/micromamba + eval "$(~/bin/micromamba shell hook -s posix)" micromamba --version # create and activate the environment - micromamba create -y -n env python=3.13 gdal natcap.invest==3.17.2 - micromamba activate env + micromamba create -y -n server_env python=3.13 + micromamba activate server_env pip install --no-cache-dir --only-binary=:all: pygeoapi pygeoapi --version diff --git a/slurm_cluster_config/startup_script.sh b/slurm_cluster_config/startup_script.sh deleted file mode 100644 index 153dc1d4..00000000 --- a/slurm_cluster_config/startup_script.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -# install micromamba -curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba -eval "$(./bin/micromamba shell hook -s posix)" -micromamba --version - -# create and activate the environment -micromamba create -y -n env python=3.13 gdal natcap.invest==3.17.2 -micromamba activate env - -# activate the environment in future shells -echo 'echo "activating micromamba env"' >> /etc/bashrc -echo 'eval "$(./bin/micromamba shell hook -s posix)" && micromamba activate env' >> /etc/bashrc diff --git a/tests/test_server.py b/tests/test_server.py index 5bc5d184..50caae1e 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -6,7 +6,6 @@ import time import unittest -from invest_processes.utils import download_and_extract_datastack from pygeoapi import flask_app CARBON_DATASTACK_URL = 'https://raw.githubusercontent.com/natcap/invest-compute/refs/heads/main/tests/test_data/invest_carbon_datastack.tgz' @@ -238,24 +237,3 @@ def testValidateProcessExecutionSync(self): 'validation_results.json' # json output from the validate command } ) - - -class UtilsTests(unittest.TestCase): - - def setUp(self): - self.workspace_dir = tempfile.mkdtemp() - - def tearDown(self): - shutil.rmtree(self.workspace_dir) - - def testDownloadAndExtractDatastack(self): - """Test utility function for downloading and extracting a datastack.""" - json_path, model_id = download_and_extract_datastack( - CARBON_DATASTACK_URL, self.workspace_dir) - self.assertEqual( - set(os.listdir(self.workspace_dir)), - {'data', 'log.txt', 'parameters.invest.json'} - ) - self.assertEqual(json_path, os.path.join( - self.workspace_dir, 'parameters.invest.json')) - self.assertEqual(model_id, 'carbon')