Add index page to the results bucket

invest_processes/src/invest_processes/slurm_manager.py

@@ -24,7 +24,7 @@
 )
 
 LOGGER = logging.getLogger(__name__)
-BUCKET_NAME = 'invest-compute-workspaces'
+BUCKET_NAME = 'results.compute.naturalcapitalalliance.org'
 STORAGE_CLIENT = storage.Client()
 BUCKET = STORAGE_CLIENT.bucket(BUCKET_NAME)
 
@@ -205,6 +205,8 @@ def get_jobs(self,
 
         jobs = []
         for line in output_lines:
+            if line == '':
+                continue
             job_id, job_status, submit_time, start_time, end_time = line.split()
             if status and job_status != status:
                 continue

slurm_cluster_config/README.md

@@ -40,74 +40,34 @@ ansible-playbook -vvv -i gcp.yml login_node_playbook.yml
 ```
 At this point, the nodes should be configured and the pygeoapi server should be running on the login node.
 
-## Setting up the API Gateway
+## Reproducibility
+As far as possible, the goal is that all infrastructure can be destroyed with `terraform destroy` and re-created with `terraform apply`. There are a few limitations:
 
-Generate the OpenAPI specification:
-```
-export PYGEOAPI_CONFIG=../invest_processes/pygeoapi-config.yml
-export PYGEOAPI_OPENAPI=openapi.yml
-pygeoapi openapi generate $PYGEOAPI_CONFIG --output-file $PYGEOAPI_OPENAPI
-```
-
-The generated openapi.yml contains external references. API Gateway doesn't like that, so we need to bundle it so all references are available locally:
-```
-npm i -g @redocly/cli@latest
-```
-
-One of the referenced OGC API specifications [is invalid](https://github.com/opengeospatial/ogcapi-processes/issues/523).
-To fix this, we have to manually edit the default value. In `bundled-openapi.yml`, replace
-```
-    transmissionMode:
-      type: string
-      enum:
-        - value
-        - reference
-      default:
-        - value
-```
-with
-```
-    transmissionMode:
-      type: string
-      enum:
-        - value
-        - reference
-      default: value
-```
-and replace
-```
-        response:
-          type: string
-          enum:
-            - raw
-            - document
-          default:
-            - raw
-```
-with
-```
-        response:
-          type: string
-          enum:
-            - raw
-            - document
-          default: raw
-```
-
-## hpc-slurm.yml
-Cluster blueprint file for use with Google Cluster Toolkit.
-Adapted from the example:
-https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/examples/hpc-slurm.yaml
+- Some GCP resources cannot be instantly deleted and re-created. A reference to certain types of resources can persist for hours or days. Because of this, it is expected that running `terraform destroy` will initially fail. It requires a two-pass approach: run `terraform destroy`, expecting an error, then run `terraform destroy` again a few hours later, at which point it should succeed in cleaning up the remaining resources.
+- In our case, the domain name we wanted to use (`compute.naturalcapitalalliance.org`) was already managed under a different GCP project, so the domain registration and DNS is not included in the terraform config. You will need to set up a DNS record pointing to the load balancer public IP address.
 
+## Relevant files
 
-## ansible.cfg
+### ansible.cfg
 Ansible configuration file. Using this to enable the the `google.cloud.gcp_compute` plugin, which dynamically inventories GCP resources.
 
-## gcp.yml
+### bundled-openapi.yml
+This file is modified from the openapi document generated by pygeoapi. The generated version contained external references, which have been bundled into `bundled-openapi.yml` using `redocly`. It also had some errors which have been fixed manually. For these reasons we are tracking `bundled-openapi.yml` rather than generating it with `pygeoapi` as needed. The `pygeoapi openapi generate` command may still be useful if we need to make changes to an endpoint.
+
+### gcp.yml
 Configures the `google.cloud.gcp_compute` plugin to dynamically find the GCP VM that we want to apply the playbook to (in this case the slurm login node).
 
 Note: this file MUST be called `gcp.yml` (or `gcp_compute.yml`) EVEN when you directly point to it with the `-i` flag. This is an undocumented requirement that I only figured out by looking at the plugin source code.
 
+### login_node_playbook.yml
+Ansible playbook file that automates the process of configuring and launching the `pygeoapi` server on the login node.
+
+### hpc-slurm/
+Terraform files that specify the GCP resources for the project.
+Adapted from the Google Cluster Toolkit slurm example: https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/examples/hpc-slurm.yaml
+
+
+
 
 
 

slurm_cluster_config/hpc-slurm/main.tf

@@ -58,7 +58,8 @@ module "homefs" {
   region            = var.region
   reserved_ip_range = module.private_service_access.reserved_ip_range
   zone              = var.zone
-  size_gb           = 2560
+  filestore_tier    = "BASIC_HDD"
+  size_gb           = 1024
 }
 
 module "debug_nodeset" {
@@ -71,7 +72,6 @@ module "debug_nodeset" {
   node_count_dynamic_max  = 4
   project_id              = var.project_id
   region                  = var.region
-  startup_script          = "/home/bin/startup_script.sh"
   subnetwork_self_link    = module.network.subnetwork_self_link
   zone                    = var.zone
 }
@@ -94,7 +94,6 @@ module "compute_nodeset" {
   node_count_dynamic_max  = 20
   project_id              = var.project_id
   region                  = var.region
-  startup_script          = "/home/bin/startup_script.sh"
   subnetwork_self_link    = module.network.subnetwork_self_link
   zone                    = var.zone
 }

slurm_cluster_config/hpc-slurm/server_interface.tf

@@ -50,6 +50,7 @@ resource "google_cloud_run_v2_service" "proxy" {
   location = var.region
   ingress  = "INGRESS_TRAFFIC_ALL" # Accessible from Gateway (Public)
   deletion_protection = false
+  depends_on = [module.network]  # wait until the network infrastructure exists
 
   template {
     service_account = google_service_account.cloud_run_sa.email
@@ -229,7 +230,8 @@ resource "google_project_service" "api_gateway_service" {
   disable_on_destroy = false
 
   # Wait for the timer to give time for the service to appear
-  depends_on = [time_sleep.wait_for_api_service_propagation]
+  # Force enablement to wait until the configuration is entirely ready
+  depends_on = [time_sleep.wait_for_api_service_propagation, google_api_gateway_api_config.api_cfg]
 }
 
 # Create the API Gateway
@@ -260,10 +262,11 @@ locals {
   }
 }
 
-
+# API keys are soft-deleted and cannot be re-created with the same name for
+# some time after deletion, so give them an unique name with a random id
 resource "google_apikeys_key" "api_keys" {
   for_each     = local.api_clients
-  name         = each.key
+  name         = "${each.key}-${random_id.suffix.hex}"
   display_name = each.value
   project      = var.project_id
 
@@ -344,8 +347,8 @@ resource "google_compute_global_address" "default" {
 #
 # Job workspaces will be uploaded to this bucket. Users can download their
 # results and logs from the bucket.
-resource "google_storage_bucket" "invest-compute-workspaces" {
-  name          = "invest-compute-workspaces"
+resource "google_storage_bucket" "results_compute_naturalcapitalalliance_org" {
+  name          = "results.compute.naturalcapitalalliance.org"
   location      = "US"
   force_destroy = true
 
@@ -358,16 +361,44 @@ resource "google_storage_bucket" "invest-compute-workspaces" {
       type = "Delete"
     }
   }
+
+  website {
+    main_page_suffix = "index.html"
+  }
 }
 
 # Make bucket public - anyone can view contents
 resource "google_storage_bucket_iam_member" "member" {
   provider = google
-  bucket   = google_storage_bucket.invest-compute-workspaces.name
+  bucket   = google_storage_bucket.results_compute_naturalcapitalalliance_org.name
   role     = "roles/storage.objectViewer"
   member   = "allUsers"
 }
 
+resource "google_storage_bucket_object" "upload_js" {
+  name           = "bucketlist.js"
+  bucket         = google_storage_bucket.results_compute_naturalcapitalalliance_org.name
+  source         = "${path.module}/results_bucket/bucketlist.js"
+  content_type   = "application/javascript"
+  source_md5hash = filemd5("${path.module}/results_bucket/bucketlist.js")
+}
+
+resource "google_storage_bucket_object" "upload_html" {
+  name           = "index.html"
+  bucket         = google_storage_bucket.results_compute_naturalcapitalalliance_org.name
+  source         = "${path.module}/results_bucket/index.html"
+  content_type   = "text/html"
+  source_md5hash = filemd5("${path.module}/results_bucket/index.html")
+}
+
+resource "google_storage_bucket_object" "upload_txt" {
+  name           = "robots.txt"
+  bucket         = google_storage_bucket.results_compute_naturalcapitalalliance_org.name
+  source         = "${path.module}/results_bucket/robots.txt"
+  content_type   = "text/plain"
+  source_md5hash = filemd5("${path.module}/results_bucket/robots.txt")
+}
+
 # -----------------------------------------------------------------------------
 # Testing Infrastructure
 
@@ -392,8 +423,14 @@ variable "github_repo" {
 }
 
 # Create Workload Identity Pool and Provider to authenticate GHA workflows
+resource "random_id" "suffix" {
+  byte_length = 4
+}
+
+# Workload Identity Pools are soft-deleted and cannot be re-created with the
+# same name for some time after deletion, so give them an unique name with a random id
 resource "google_iam_workload_identity_pool" "github_actions" {
-  workload_identity_pool_id = "github-actions-pool"
+  workload_identity_pool_id = "github-actions-pool-${random_id.suffix.hex}"
 }
 
 resource "google_iam_workload_identity_pool_provider" "github_actions" {
@@ -436,3 +473,63 @@ output "load_balancer_ip" {
   description = "The public IP address of the global load balancer"
   value       = google_compute_global_forwarding_rule.default.ip_address
 }
+
+
+# -----------------------------------------------------------------------------
+# Client datastack bucket
+#
+# Clients with access can upload their datastack to this bucket to make it
+# available to the invest-compute server.
+resource "google_storage_bucket" "invest_compute_client_datastacks" {
+  name          = "invest_compute_client_datastacks"
+  location      = "US"
+  force_destroy = true
+
+  # delete contents older than 3 days
+  lifecycle_rule {
+    condition {
+      age = 3
+    }
+    action {
+      type = "Delete"
+    }
+  }
+}
+
+# Make bucket public - anyone can view contents
+resource "google_storage_bucket_iam_member" "client_datastacks_member" {
+  provider = google
+  bucket   = google_storage_bucket.invest_compute_client_datastacks.name
+  role     = "roles/storage.objectViewer"
+  member   = "allUsers"
+}
+
+
+# -----------------------------------------------------------------------------
+# invest compute plugin testing service account
+# allows uploading to client datastacks bucket
+
+# Create the Service Account
+resource "google_service_account" "plugin_testing_uploader" {
+  account_id   = "plugin-testing-uploader"
+  project      = var.project_id
+}
+
+# Bind the IAM role to the client datastacks bucket
+resource "google_storage_bucket_iam_member" "bucket_upload_access" {
+  bucket = google_storage_bucket.invest_compute_client_datastacks.name
+  role   = "roles/storage.objectCreator"
+  member = "serviceAccount:${google_service_account.plugin_testing_uploader.email}"
+}
+
+# Generate a JSON key file
+resource "google_service_account_key" "plugin_testing_uploader_key" {
+  service_account_id = google_service_account.plugin_testing_uploader.name
+  public_key_type    = "TYPE_X509_PEM_FILE"
+}
+
+output "plugin_testing_service_account_json" {
+  description = "JSON key for service account needed for invest-compute plugin testing"
+  value       = base64decode(google_service_account_key.plugin_testing_uploader_key.private_key)
+  sensitive   = true
+}

slurm_cluster_config/login_node_playbook.yml

@@ -42,25 +42,31 @@
           - "--exclude=slurm_cluster_config"
           - "--exclude=tests"
 
-    - name: Move startup script to shared location
-      become: true
-      ansible.builtin.shell: |
-        mkdir /home/bin
-        mv /home/esoth_stanford_edu/invest-compute/invest_processes/startup_script.sh /home/bin
-        chmod +x /home/bin/startup_script.sh
 
     - name: Install and launch server
       args:
         chdir: invest-compute/invest_processes
         executable: /bin/bash # Ensure bash is used for shell hooks
       ansible.builtin.shell: |
         # install micromamba
-        rm -f bin/micromamba
-        curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj ~/bin/micromamba
+        rm -f ~/bin/micromamba
+        curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xj -C ~ bin/micromamba
         eval "$(~/bin/micromamba shell hook -s posix)"
         micromamba --version
 
-        # create and activate the environment
+        # create the environment for running invest
+        # this is created on the filestore so it is shared among all nodes
+        # this saves time on node startup - nodes do not have to create their own envs
+        source /opt/rh/gcc-toolset-12/enable
+        micromamba create -y -n invest_env python=3.13 gdal
+        micromamba activate invest_env
+        pip install setuptools setuptools_scm build cython babel
+        pip install --no-build-isolation git+https://github.com/natcap/pygeoprocessing.git
+        pip install --no-build-isolation git+https://github.com/emlys/invest.git@bugfix/2572
+        invest --version
+        micromamba deactivate
+
+        # create and activate the server environment
         micromamba create -y -n server_env python=3.13
         micromamba activate server_env
         pip install --no-cache-dir --only-binary=:all: pygeoapi