Add index page to the results bucket

.github/workflows/test.yml

@@ -154,7 +154,7 @@ jobs:
       - uses: 'google-github-actions/auth@v3'
         with:
           service_account: 'github-actions-sa@sdss-sdss-invest-compute.iam.gserviceaccount.com'
-          workload_identity_provider: 'projects/616945215462/locations/global/workloadIdentityPools/github-actions-pool-910c7e51/providers/github-actions'
+          workload_identity_provider: 'projects/616945215462/locations/global/workloadIdentityPools/github-actions-pool-c45705f1/providers/github-actions'
 
       - name: Install package
         run: cd invest_processes && pip install .

invest_processes/src/invest_processes/slurm_manager.py

@@ -24,7 +24,7 @@
 )
 
 LOGGER = logging.getLogger(__name__)
-BUCKET_NAME = 'invest-compute-workspaces'
+BUCKET_NAME = 'results.compute.naturalcapitalalliance.org'
 STORAGE_CLIENT = storage.Client()
 BUCKET = STORAGE_CLIENT.bucket(BUCKET_NAME)
 
@@ -205,6 +205,8 @@ def get_jobs(self,
 
         jobs = []
         for line in output_lines:
+            if line == '':
+                continue
             job_id, job_status, submit_time, start_time, end_time = line.split()
             if status and job_status != status:
                 continue
@@ -419,7 +421,11 @@ def get_job_result(self, job_id: str) -> Tuple[str, Any]:
         :returns: `tuple` of mimetype and raw output
         """
         job_info = self.get_job(job_id)
-        return job_info["mimetype"], {'workspace_url': job_info["location"]}
+        workspace_name = job_info['location'].split('/')[-1]
+        return job_info['mimetype'], {
+            'workspace_url': job_info['location'],
+            'bucket_url': f'http://results.compute.naturalcapitalalliance.org/?prefix={workspace_name}/'
+        }
 
     def delete_job(self, job_id: str) -> bool:
         """
@@ -541,10 +547,11 @@ def monitor_job_status(self, job_id, workspace_dir, get_outputs_func):
                 # get a dict of outputs (if any) from the workspace
                 # for the validate process, this includes the validation messages
                 # for the execute process, this is an empty dictionary
+                outputs = get_outputs_func(workspace_dir)
                 # include the workspace url in the outputs for all processes
                 # this is only used in sync mode to return results immediately
-                outputs = get_outputs_func(workspace_dir)
-                outputs['workspace_url'] = f'gs://{BUCKET_NAME}/{Path(workspace_dir).name}'
+                _, default_outputs = self.get_job_result(job_id)
+                outputs.update(default_outputs)
 
                 # Upload the workspace even if something went wrong, so that the
                 # user can access the slurm related files and any partial results.

invest_processes/templates/jobs/results/index.html

@@ -0,0 +1,24 @@
+{% extends "_base.html" %}
+{% block title %}{{ super() }} {% trans %}Job result{% endtrans %} {% endblock %}
+{% block crumbs %}{{ super() }}
+/ <a href="../../jobs">{% trans %}Jobs{% endtrans %}</a>
+/ <a href="../{{ data.job.id }}">{{ data.job.id }}</a>
+/ <a href="./results">{% trans %}Results{% endtrans %}</a>
+{% endblock %}
+{% block body %}
+  <section id="jobs">
+    <h2>{% trans %}Results of job{% endtrans %} {{data.job.id}}</h2>
+    <div>
+      Browse results at:
+      <pre>
+        <a href="{{ data.result.bucket_url }}">{{ data.result.bucket_url }}</a>
+      </pre>
+    </div>
+    <div>
+      Download the entire workspace:
+      <pre>
+        gcloud storage cp --recursive "{{ data.result.workspace_url }}/*" .
+      </pre>
+    </div>
+  </section>
+{% endblock %}

slurm_cluster_config/README.md

@@ -40,74 +40,34 @@ ansible-playbook -vvv -i gcp.yml login_node_playbook.yml
 ```
 At this point, the nodes should be configured and the pygeoapi server should be running on the login node.
 
-## Setting up the API Gateway
+## Reproducibility
+As far as possible, the goal is that all infrastructure can be destroyed with `terraform destroy` and re-created with `terraform apply`. There are a few limitations:
 
-Generate the OpenAPI specification:
-```
-export PYGEOAPI_CONFIG=../invest_processes/pygeoapi-config.yml
-export PYGEOAPI_OPENAPI=openapi.yml
-pygeoapi openapi generate $PYGEOAPI_CONFIG --output-file $PYGEOAPI_OPENAPI
-```
-
-The generated openapi.yml contains external references. API Gateway doesn't like that, so we need to bundle it so all references are available locally:
-```
-npm i -g @redocly/cli@latest
-```
-
-One of the referenced OGC API specifications [is invalid](https://github.com/opengeospatial/ogcapi-processes/issues/523).
-To fix this, we have to manually edit the default value. In `bundled-openapi.yml`, replace
-```
-    transmissionMode:
-      type: string
-      enum:
-        - value
-        - reference
-      default:
-        - value
-```
-with
-```
-    transmissionMode:
-      type: string
-      enum:
-        - value
-        - reference
-      default: value
-```
-and replace
-```
-        response:
-          type: string
-          enum:
-            - raw
-            - document
-          default:
-            - raw
-```
-with
-```
-        response:
-          type: string
-          enum:
-            - raw
-            - document
-          default: raw
-```
-
-## hpc-slurm.yml
-Cluster blueprint file for use with Google Cluster Toolkit.
-Adapted from the example:
-https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/examples/hpc-slurm.yaml
+- Some GCP resources cannot be instantly deleted and re-created. A reference to certain types of resources can persist for hours or days. Because of this, it is expected that running `terraform destroy` will initially fail. It requires a two-pass approach: run `terraform destroy`, expecting an error, then run `terraform destroy` again a few hours later, at which point it should succeed in cleaning up the remaining resources.
+- In our case, the domain name we wanted to use (`compute.naturalcapitalalliance.org`) was already managed under a different GCP project, so the domain registration and DNS is not included in the terraform config. You will need to set up a DNS record pointing to the load balancer public IP address.
 
+## Relevant files
 
-## ansible.cfg
+### ansible.cfg
 Ansible configuration file. Using this to enable the the `google.cloud.gcp_compute` plugin, which dynamically inventories GCP resources.
 
-## gcp.yml
+### bundled-openapi.yml
+This file is modified from the openapi document generated by pygeoapi. The generated version contained external references, which have been bundled into `bundled-openapi.yml` using `redocly`. It also had some errors which have been fixed manually. For these reasons we are tracking `bundled-openapi.yml` rather than generating it with `pygeoapi` as needed. The `pygeoapi openapi generate` command may still be useful if we need to make changes to an endpoint.
+
+### gcp.yml
 Configures the `google.cloud.gcp_compute` plugin to dynamically find the GCP VM that we want to apply the playbook to (in this case the slurm login node).
 
 Note: this file MUST be called `gcp.yml` (or `gcp_compute.yml`) EVEN when you directly point to it with the `-i` flag. This is an undocumented requirement that I only figured out by looking at the plugin source code.
 
+### login_node_playbook.yml
+Ansible playbook file that automates the process of configuring and launching the `pygeoapi` server on the login node.
+
+### hpc-slurm/
+Terraform files that specify the GCP resources for the project.
+Adapted from the Google Cluster Toolkit slurm example: https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/examples/hpc-slurm.yaml
+
+
+
 
 
 

slurm_cluster_config/hpc-slurm/main.tf

@@ -58,7 +58,8 @@ module "homefs" {
   region            = var.region
   reserved_ip_range = module.private_service_access.reserved_ip_range
   zone              = var.zone
-  size_gb           = 2560
+  filestore_tier    = "BASIC_HDD"
+  size_gb           = 1024
 }
 
 module "debug_nodeset" {
@@ -71,7 +72,6 @@ module "debug_nodeset" {
   node_count_dynamic_max  = 4
   project_id              = var.project_id
   region                  = var.region
-  startup_script          = "/home/bin/startup_script.sh"
   subnetwork_self_link    = module.network.subnetwork_self_link
   zone                    = var.zone
 }
@@ -94,7 +94,6 @@ module "compute_nodeset" {
   node_count_dynamic_max  = 20
   project_id              = var.project_id
   region                  = var.region
-  startup_script          = "/home/bin/startup_script.sh"
   subnetwork_self_link    = module.network.subnetwork_self_link
   zone                    = var.zone
 }