diff --git a/Makefile b/Makefile index ce7694d6e..5c14e9fbe 100644 --- a/Makefile +++ b/Makefile @@ -4,14 +4,20 @@ DEMO_MODE := 0 FORCE := 0 +# Repository root. Defined before the tool detection below, which depends on it. +ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) + # Required System files -DOCKER_COMPOSE_EXE := $(shell which docker-compose) +# Prefer the Compose V2 plugin ("docker compose"), falling back to the standalone +# V1 binary. V1 is no longer shipped with current Docker Desktop releases. +DOCKER_COMPOSE_EXE := $(shell docker compose version >/dev/null 2>&1 && echo "docker compose" || which docker-compose) CURL_EXE := $(shell which curl) -MVN_EXE := $(shell which mvn) +# Prefer the Maven wrapper bundled with this repository, so a system-wide Maven +# install is not required. +MVN_EXE := $(shell if [ -x $(ROOT_DIR)/mvnw ]; then echo $(ROOT_DIR)/mvnw; else which mvn; fi) # Variables DOCKERFILE_NAME := $(shell if [ $(DEMO_MODE) -eq 1 ]; then echo Dockerfile; else echo Dockerfile.dev; fi) -ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) MY_UID := $$(id -u) MY_GID := $$(id -g) THIS_USER := $(MY_UID):$(MY_GID) diff --git a/README.md b/README.md index 9d7f8176e..0e43b598f 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,9 @@ Song functions as a file catalog system, tracking files and managing their metad Technical resources for those working with or contributing to the project are available from our official documentation site, the following content can also be read and updated within the `/docs` folder of this repository. -- **[Song Overview](https://docs.overture.bio/docs/core-software/Song/overview)** -- [**Setting up the Development Enviornment**](https://docs.overture.bio/docs/core-software/Song/setup) -- [**Common Usage Docs**](https://docs.overture.bio/docs/core-software/Song/setup) +- **[Song Overview](https://docs.overture.bio/develop/Song/overview)** +- [**Setting up the Development Enviornment**](https://docs.overture.bio/develop/Song/setup) +- [**Common Usage Docs**](https://docs.overture.bio/develop/Song/setup) ## Development Environment @@ -32,7 +32,7 @@ Technical resources for those working with or contributing to the project are av - For support, feature requests, and bug reports, please see our [Support Guide](https://docs.overture.bio/community/support). -- For detailed information on how to contribute to this project, please see our [Contributing Guide](https://docs.overture.bio/docs/contribution). +- For detailed information on how to contribute to this project, please see our [Contributing Guide](https://docs.overture.bio/develop/contributing). ## Related Software @@ -50,7 +50,7 @@ The Overture Platform includes the following Overture Components: |[Lyric](https://github.com/overture-stack/lyric)| A model-agnostic, tabular data submission system | |[Lectern](https://github.com/overture-stack/lectern)| Schema Manager, designed to validate, store, and manage collections of data dictionaries. | -If you'd like to get started using our platform [check out our quickstart guides](https://docs.overture.bio/guides/getting-started) +If you'd like to get started using our platform [check out our quickstart guides](https://docs.overture.bio/deploy) ## Funding Acknowledgement diff --git a/docker-compose.yml b/docker-compose.yml index e405b8db8..da1d05dae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,9 @@ version: "3.7" services: keycloak-server: - image: docker.io/bitnami/keycloak:22 + # Bitnami moved its versioned Docker Hub tags to the bitnamilegacy archive in 2025; + # docker.io/bitnami/keycloak:22 no longer resolves. See DEVELOPMENT notes before bumping. + image: docker.io/bitnamilegacy/keycloak:22 environment: - KC_DB=postgres - KC_DB_URL=jdbc:postgresql://keycloak-postgresql/bitnami_keycloak @@ -23,7 +25,7 @@ services: curl -sL https://github.com/oicr-softeng/keycloak-apikeys/releases/download/1.0.1/keycloak-apikeys-1.0.1.jar -o /opt/bitnami/keycloak/providers/keycloak-apikeys-1.0.1.jar kc.sh start-dev --import-realm keycloak-postgresql: - image: docker.io/bitnami/postgresql:11 + image: docker.io/bitnamilegacy/postgresql:11 environment: # ALLOW_EMPTY_PASSWORD is recommended only for development. - ALLOW_EMPTY_PASSWORD=yes diff --git a/docs/00-overview.md b/docs/00-overview.md index eb931bb03..384257225 100644 --- a/docs/00-overview.md +++ b/docs/00-overview.md @@ -31,7 +31,7 @@ As part of the larger Overture.bio software suite, Song is typically used with a . ├── /song-client ├── /song-core -└── /song-servers +└── /song-server ``` [Click here to view the Song respository on GitHub ](https://github.com/overture-stack/song) @@ -62,7 +62,6 @@ These packages are: - **song-docker-demo**: Example all-in-one deployment of Song with Score with other external project Auth and ID services. - **song-docs**: Deprecated `readthedocs` documentation site - **song-go-client**: Alternate cli imlementation, written in GoLang. -- **song-java-sdk**: Code library for java applications to interact with a Song server programatically. -- **song-python-sdk**: Code library for java applications to interact with a Song server programatically. +- **song-python-sdk**: Code library for Python applications to interact with a Song server programatically. diff --git a/docs/01-setup.md b/docs/01-setup.md index 35efc39cd..141721b64 100644 --- a/docs/01-setup.md +++ b/docs/01-setup.md @@ -13,55 +13,59 @@ This guide will walk you through setting up a complete development environment, ### Setting up supporting services -We'll use our Quickstart service, a flexible Docker Compose setup, to spin up Song's complementary services. +The Song repository ships its own `docker-compose.yml` and `Makefile`, which together start every service Song depends on. No other repository is required. -1. Clone the Quickstart repository and move into its directory: +1. Clone Song and move into its directory: ```bash - git clone -b quickstart https://github.com/overture-stack/prelude.git - cd prelude + git clone https://github.com/overture-stack/song.git + cd song ``` -2. Run the appropriate start command for your operating system: +2. Start Song's dependencies: - | Operating System | Command | - | ---------------- | -------------------- | - | Unix/macOS | `make SongDev` | - | Windows | `./make.bat SongDev` | + ```bash + make start-deps + ```
**Click here for a detailed breakdown** - This command will set up all complementary services for Song development as follows: - - ![SongDev](./assets/songDev.svg "Song Dev Environment") + `make start-deps` packages the project and then brings up Keycloak, Score, and object storage from the repository's `docker-compose.yml`: | Service | Port | Description | Purpose in Song Development | | ----------- | ------ | ----------------------------------------------- | ------------------------------------------------------ | - | Conductor | `9204` | Orchestrates deployments and environment setups | Manages the overall development environment | - | Keycloak-db | - | Database for Keycloak (no exposed port) | Stores Keycloak data for authentication | - | Keycloak | `8180` | Authorization and authentication service | Provides OAuth2 authentication for Score | - | Song-db | `5433` | Database for Song | Stores metadata managed by Song | + | Keycloak | `9082` | Authorization and authentication service | Provides OAuth2 authentication for Song | + | Keycloak-db | `9444` | Database for Keycloak | Stores Keycloak data for authentication | | Score | `8087` | File Transfer service | Handles file uploads, downloads, and storage operation | - | Minio | `9000` | Object storage provider | Simulates S3-compatible storage for Score | + | Minio | `8085` | Object storage provider | Simulates S3-compatible storage for Score | + + Keycloak starts with the `myrealm` realm imported from `docker/keycloak-init/data_import`, and downloads the `keycloak-apikeys` provider on start-up so it can issue API keys. + + To bring up Song itself along with its database and all of the above, use `make start-song-server` instead. That adds: + + | Service | Port | Description | Purpose in Song Development | + | ----------- | -------------- | --------------------- | -------------------------------- | + | Song-db | `8432` | Database for Song | Stores metadata managed by Song | + | Song-server | `8080`, `5006` | The Song server | The service under development; `5006` is the JVM debug port | - Ensure these ports are free on your system before starting the environment. - You may need to adjust the ports in the `docker-compose.yml` file if you have conflicts with existing services. + - `make clean` tears the stack down and removes the build output; `make log-song-server` tails the server's logs. + + :::note - For more information, see our [Quickstart documentation linked here](https://docs.overture.bio/docs/other-software/Quickstart) + These targets build the project with the bundled Maven wrapper and drive Docker Compose, so a JDK is required even when you only want the supporting services. See the prerequisites above. + + :::
### Running the Development Server -1. Clone Song and move into its directory: - - ```bash - git clone https://github.com/overture-stack/song.git - cd song - ``` +Use these steps to run Song on your host, against the supporting services started above. To run Song in a container instead, `make start-song-server` covers both. -2. Build the application locally: +1. Build the application locally: ```bash ./mvnw clean install -DskipTests @@ -88,7 +92,7 @@ We'll use our Quickstart service, a flexible Docker Compose setup, to spin up So ::: -3. Start the Song Server: +2. Start the Song Server: ```bash ./mvnw spring-boot:run -Dspring-boot.run.profiles=default,dev,secure -pl song-server @@ -96,7 +100,7 @@ We'll use our Quickstart service, a flexible Docker Compose setup, to spin up So :::info - If you are looking to configure Song for your specific environment, [**the Song-server configuration file can be found here**](https://github.com/overture-stack/score/blob/develop/score-server/src/main/resources/application.yml). A summary of the available Spring profiles is provided below: + If you are looking to configure Song for your specific environment, [**the Song-server configuration file can be found here**](https://github.com/overture-stack/song/blob/develop/song-server/src/main/resources/application.yml). A summary of the available Spring profiles is provided below:
**Click here for a summary of the Song-server spring profiles** @@ -143,21 +147,24 @@ After installing and configuring Song, verify that the system is functioning cor - Verify you're using the correct URL 3. **Test GET Analysis Endpoint** + + This step needs a study to query. The repository's Compose stack starts with an empty database, so create one first (see [Data model management](/develop/Song/Reference/data-model-management)) and substitute its ID for `` below. + - Using Swagger UI: 1. Locate the `GetAnalysesForStudy` endpoint in the **Analysis** section: `GET /studies/{studyId}/analysis/paginated` 2. Click to expand and select "Try it out" 3. Set parameters: - analysisStates: PUBLISHED - - studyId: demo + - studyId: `` 4. Click "Execute" - Alternatively, use curl: ```bash - curl -X GET "http://localhost:8080/studies/demo/analysis?analysisStates=PUBLISHED" -H "accept: */*" + curl -X GET "http://localhost:8080/studies//analysis?analysisStates=PUBLISHED" -H "accept: */*" ``` - - Expected result: JSON response containing analysis data for the demo study + - Expected result: JSON response containing the analyses registered under that study :::info Need Help? -If you encounter any issues or have questions about our API, please don't hesitate to reach out through our relevant [**community support channels**](https://docs.overture.bio/community/support). +If you encounter any issues or have questions about our API, please don't hesitate to reach out through our [**support page**](/community/support) or our [**discussion forum**](https://github.com/overture-stack/docs/discussions?discussions_q=). ::: ## Song-Client Setup @@ -166,8 +173,8 @@ The `song-client` is a CLI tool used for communicating with a `song-server`. For ```bash docker run -d --name song-client \ - -e CLIENT_ACCESS_TOKEN=68fb42b4-f1ed-4e8c-beab-3724b99fe528 \ - -e CLIENT_STUDY_ID=demo \ + -e CLIENT_ACCESS_TOKEN= \ + -e CLIENT_STUDY_ID= \ -e CLIENT_SERVER_URL=http://localhost:8080 \ --network="host" \ --platform="linux/amd64" \ @@ -175,10 +182,58 @@ docker run -d --name song-client \ ghcr.io/overture-stack/song-client:5.1.1 \ ``` +:::info Obtaining an API key + +`CLIENT_ACCESS_TOKEN` is environment-specific; there is no fixed development token. The Keycloak that `make start-deps` brings up on port `9082` loads the `keycloak-apikeys` provider, which issues keys against the `myrealm` realm. See [Authentication](/develop/Song/Reference/authentication) for how the provider is installed and how Song validates the keys it issues. + +
+**Click here for the steps to generate a key against the local stack** + +The realm ships the users `admin` (a member of the `ADMIN` group) and `testca_user` (a member of `TESTCASONG_GROUP`), both with hashed passwords that are not recoverable from the realm export. Keys can only be issued by their owner or an administrator, so start by giving one of those users a password you know. + +1. Open the Keycloak admin console at `http://localhost:9082` and sign in. The image's default administrator credentials are `user` / `bitnami`. + +2. In the `myrealm` realm, set a password for the `admin` user (**Users** → `admin` → **Credentials**). Note its user ID from the same page; you will need it below. + +3. Request a token for that user. The realm's `system` client has direct access grants enabled: + + ```bash + curl -X POST "http://localhost:9082/realms/myrealm/protocol/openid-connect/token" \ + -d "grant_type=password" \ + -d "client_id=system" -d "client_secret=systemsecret" \ + -d "username=admin" -d "password=" + ``` + +4. Exchange that token for an API key, substituting the user ID from step 2: + + ```bash + curl -X POST "http://localhost:9082/realms/myrealm/apikey/api_key?user_id=&scopes=song.WRITE&scopes=score.WRITE" \ + -H "Authorization: Bearer " + ``` + + The `name` field of the response is the key value. Pass it as `CLIENT_ACCESS_TOKEN`: + + ```json + { + "name": "5b1da354-37bd-409d-b938-ea14b8035bc3", + "scope": ["score.WRITE", "song.WRITE"], + "expiryDate": "2027-07-30T15:32:59.990+0000", + "isRevoked": false + } + ``` + +Scopes take the form `.`, and the resources the realm defines are `song`, `score`, `TEST-CA`, and `ABC123`. A request for a scope the user's group does not carry is rejected with `Invalid Scope`. + +
+ +`CLIENT_STUDY_ID` must name a study that already exists on your server. The repository's Compose stack starts with an empty database, so create one first; see [Data model management](/develop/Song/Reference/data-model-management). + +::: +
**Click here for an explaination of command above** - - `-e CLIENT_ACCESS_TOKEN=68fb42b4-f1ed-4e8c-beab-3724b99fe528` sets up the song-client with a pre-configured system-wide access token that works with the conductor service setup. - - `-e CLIENT_STUDY_ID=demo` the quickstart is pre-configured with a Study ID named demo, we supply the Study ID value to the song-client on start-up. + - `-e CLIENT_ACCESS_TOKEN=` supplies the API key the song-client authenticates with, obtained from Keycloak as described above. + - `-e CLIENT_STUDY_ID=` the Study ID the song-client operates against, supplied on start-up. - `-e CLIENT_SERVER_URL=http://localhost:8080` is the url for the Song server which the Song-Client will interact with. - `--network="host"` Uses the host network stack inside the container, bypassing the usual network isolation. This means the container shares the network namespace with the host machine. - `--platform="linux/amd64"` Specifies the platform the container should emulate. In this case, it's set to linux/amd64, indicating the container is intended to run on a Linux system with an AMD64 architecture. diff --git a/docs/02-Usage/00-submitting-metadata.md b/docs/02-Reference/00-submitting-metadata.md similarity index 89% rename from docs/02-Usage/00-submitting-metadata.md rename to docs/02-Reference/00-submitting-metadata.md index ce172f397..0cca881c3 100644 --- a/docs/02-Usage/00-submitting-metadata.md +++ b/docs/02-Reference/00-submitting-metadata.md @@ -1,9 +1,9 @@ # Metadata Submission -Submitting new metadata entries ([Analyses](https://docs.overture.bio/docs/core-software/Song/Usage/#song-terminology)) to Song. +Submitting new metadata entries ([Analyses](/develop/Song/Reference/#song-terminology)) to Song. :::info CLI Submission Guide -This page documents the basic submission flow, for a more detailed guide see our [**platform guide on CLI submissions**](https://docs.overture.bio/guides/user-guides/cli-submissions). +This page documents the basic submission flow, for a more detailed guide see our [**platform guide on CLI submissions**](/use/cli-submissions). ::: ### Client Installation @@ -80,7 +80,7 @@ This page documents the basic submission flow, for a more detailed guide see our ### Step 1: Prepare a Payload -Create a metadata payload conforming to a registered `analysis_type` schema. For more information, see our section covering [Data Model Management](https://docs.overture.bio/docs/core-software/Song/Usage/data-model-management). +Create a metadata payload conforming to a registered `analysis_type` schema. For more information, see our section covering [Data Model Management](/develop/Song/Reference/data-model-management). ### Step 2: Upload the Metadata Payload @@ -138,7 +138,7 @@ Set the analysis state to `PUBLISHED`: docker exec song-client sh -c "sing publish -a a4142a01-1274-45b4-942a-01127465b422" ``` -For more information, see our documentation on [**Song publication controls**](https://docs.overture.bio/docs/core-software/Song/Usage/publication-controls) +For more information, see our documentation on [**Song publication controls**](/develop/Song/Reference/publication-controls) :::info Search & Exploration For optimal data querying, use Song with our search & exploration services Maestro, Arranger and Stage. With these components you can index and query and explore Song data through an intuitive search portal. @@ -149,5 +149,5 @@ For optimal data querying, use Song with our search & exploration services Maest - If you encounter connection or internal server errors, have your admin verify that the Song and Score servers are correctly configured. :::info Support -For technical support, please don't hesitate to reach out through our relevant [**community support channels**](https://docs.overture.bio/community/support). +For technical support, please don't hesitate to reach out through our [**support page**](/community/support) or our [**discussion forum**](https://github.com/overture-stack/docs/discussions?discussions_q=). ::: diff --git a/docs/02-Usage/01-retrieving-metadata.md b/docs/02-Reference/01-retrieving-metadata.md similarity index 73% rename from docs/02-Usage/01-retrieving-metadata.md rename to docs/02-Reference/01-retrieving-metadata.md index 52524904b..b00089621 100644 --- a/docs/02-Usage/01-retrieving-metadata.md +++ b/docs/02-Reference/01-retrieving-metadata.md @@ -22,29 +22,31 @@ Swagger UI is ideal for exploration and simple use cases. It provides detailed d Here are examples of how to retrieve analyses using the Song API programmatically. -### Example 1: Sample ID Query +### Example 1: Retrieve a Single Analysis by ID -This Python script searches for associated analyses based on a specific sample ID: +When you already know an analysis's ID, retrieve it directly with the `GetAnalysis` endpoint: ```python import requests url = "https://song.virusseq-dataportal.ca" -sample = "QC_546060" study = "LSPQ-QC" +analysis_id = "4861730f-db75-4dbf-a173-0fdb75cdbfee" -endpoint = f"{url}/studies/{study}/analysis/search/id?submitterSampleId={sample}" +endpoint = f"{url}/studies/{study}/analysis/{analysis_id}" response = requests.get(endpoint) -print(response.json()[0]) +print(response.json()) ``` +To locate analyses by a submitter field (such as a sample or donor ID) rather than by analysis ID, see [Example 2](#example-2-bulk-analysis-query): Song has no server-side search for those fields, so you retrieve the study's analyses and filter them client-side. + :::info Support -For technical support or specific use cases, please don't hesitate to reach out through our relevant [**community support channels**](https://docs.overture.bio/community/support). +For technical support or specific use cases, please don't hesitate to reach out through our [**support page**](/community/support) or our [**discussion forum**](https://github.com/overture-stack/docs/discussions?discussions_q=). ::: ### Example 2: Bulk Analysis Query -This script demonstrates how to retrieve an aggregated list of published analyses and create a filtered list of analyses associated with a series of `submitterIDs`: +This is the pattern for finding analyses by a submitter field (here, a submitter sample ID). Song has no server-side search for donor, specimen, or sample IDs — they are custom-schema fields in the analysis payload, not base-schema entities — so you retrieve the study's published analyses in bulk and filter them client-side: ```python import requests @@ -60,7 +62,7 @@ status = "PUBLISHED" url = "https://song.virusseq-dataportal.ca" # Read CSV to identify specimens -tmp = pd.read_csv("/Users/esu/Desktop/GitHub/virus-seq/2023_05_31/DP_Update_consensus_seq_version.csv", sep=",") +tmp = pd.read_csv("path/to/consensus_seq_version.csv", sep=",") # Clean up CSV tmp.set_index('Specimen Collector Sample ID', inplace=True) @@ -97,7 +99,8 @@ for offset in range(100, total, 100): aggregated_analyses.extend(response.json()['analyses']) -# Filter analyses according to sample specimen ID +# Filter analyses by submitter sample ID. +# `samples[].submitterSampleId` is a custom-schema field here; adjust the path to match your schema. print("Filtering analyses") filtered_analyses = [analysis for analysis in aggregated_analyses if analysis['samples'][0]['submitterSampleId'] in tmp.index.values] diff --git a/docs/02-Usage/02-Usage.mdx b/docs/02-Reference/02-Reference.mdx similarity index 95% rename from docs/02-Usage/02-Usage.mdx rename to docs/02-Reference/02-Reference.mdx index 514bf7f00..a4091b75e 100644 --- a/docs/02-Usage/02-Usage.mdx +++ b/docs/02-Reference/02-Reference.mdx @@ -1,9 +1,6 @@ -# Usage +# Reference - - - -The following usage docs provide instructions for common tasks and workflows in Song: +The following reference docs provide instructions for common tasks and workflows in Song:

@@ -32,7 +29,7 @@ Whether you're new to these tools or looking to refresh your understanding, this - **Score**: A file transfer and storage system designed to work alongside Song for managing large data files. - **Song**: A metadata management system used to track and manage metadata related to files in a data repository. -s + - **Study**: A collection of analyses in Song, typically representing a coherent set of data for a particular research project or data submission. - **Published Analysis**: An analysis in Song that has been marked as ready for downstream processing and is searchable within the system. diff --git a/docs/02-Usage/02-updating-metadata.md b/docs/02-Reference/02-updating-metadata.md similarity index 52% rename from docs/02-Usage/02-updating-metadata.md rename to docs/02-Reference/02-updating-metadata.md index 6221b0a1e..5a7cae2df 100644 --- a/docs/02-Usage/02-updating-metadata.md +++ b/docs/02-Reference/02-updating-metadata.md @@ -19,7 +19,7 @@ Required inputs for the **PatchUpdateAnalysis** endpoint: The following examples demonstrate how to update analyses using the Song API programmatically. :::info Support -For technical support or specific use cases, please don't hesitate to reach out through our relevant [**community support channels**](https://docs.overture.bio/community/support). +For technical support or specific use cases, please don't hesitate to reach out through our [**support page**](/community/support) or our [**discussion forum**](https://github.com/overture-stack/docs/discussions?discussions_q=). ::: ### Updating a Single Analysis (Example) @@ -51,7 +51,7 @@ if patch_response.status_code != 200: ### Updating Multiple Analyses (Example) -This script updates multiple analyses associated with specific samples within a study: +This script updates every analysis whose submitter sample ID appears in a target list. Song has no server-side search for submitter sample or donor IDs, so it retrieves the study's analyses in bulk (as in the [Metadata Retrieval](./01-retrieving-metadata.md) guide) and filters them client-side before patching each match by its analysis ID: ```python import requests @@ -60,41 +60,40 @@ import requests samples_to_mod = ["SAMPLE_A", "SAMPLE_B", "SAMPLE_C", "SAMPLE_D", "SAMPLE_E", "SAMPLE_F"] url = "https://song.virusseq-dataportal.ca" study = "ABC123" +status = "PUBLISHED" api_token = "YOUR_API_TOKEN" +new_value = "Illumina MiSeq" -for sample in samples_to_mod: - # Retrieve analyses associated with samples - endpoint = f"{url}/studies/{study}/analysis/search/id?submitterDonorId={sample}" - headers = {"accept": "*/*"} +patch_headers = { + "accept": "*/*", + "Authorization": f"Bearer {api_token}", + "Content-Type": "application/json", +} - get_response = requests.get(endpoint, headers=headers) - - if get_response.status_code != 200: - print(f"Error: {endpoint}") +# Retrieve the study's analyses in bulk (paginated). +analyses = [] +offset = 0 +while True: + endpoint = f"{url}/studies/{study}/analysis/paginated?analysisStates={status}&limit=100&offset={offset}" + response = requests.get(endpoint) + if response.status_code != 200: + print(f"Error: {response.status_code}, {endpoint}") + break + page = response.json() + analyses.extend(page["analyses"]) + offset += 100 + if offset >= page["totalAnalyses"]: break - published_analyses = [analysis for analysis in get_response.json() if analysis['analysisState'] == 'PUBLISHED'] - - if len(published_analyses) == 0: - print(f"No published analysis detected: {endpoint}") - continue - if len(published_analyses) > 1: - print(f"Multiple analyses detected: {endpoint}") +# Update every analysis whose submitter sample ID is in the target list. +# `samples[].submitterSampleId` is a custom-schema field; adjust the path to match your schema. +for analysis in analyses: + if analysis["samples"][0]["submitterSampleId"] not in samples_to_mod: continue - - analysis = published_analyses[0] - analysis_id = analysis['analysisId'] - old_value = analysis['experiment']['sequencing_instrument'] - new_value = "Illumina MiSeq" - - # PATCH endpoint + + analysis_id = analysis["analysisId"] patch_endpoint = f"{url}/studies/{study}/analysis/{analysis_id}" - patch_headers = { - "accept": "*/*", - "Authorization": f"Bearer {api_token}", - "Content-Type": "application/json" - } - payload = {'experiment': {'sequencing_instrument': new_value}} + payload = {"experiment": {"sequencing_instrument": new_value}} patch_response = requests.patch(patch_endpoint, json=payload, headers=patch_headers) if patch_response.status_code != 200: @@ -104,4 +103,4 @@ for sample in samples_to_mod: ## Important Note on Song-assigned IDs -Song-assigned IDs (donor, sample, specimen, analysis, and object IDs) and those specified in the [base schema](https://github.com/overture-stack/SONG/blob/develop/song-server/src/main/resources/schemas/analysis/analysisBase.json) are immutable and cannot be altered. If you need to change any of these values, it is recommended to UNPUBLISH and SUPPRESS the analysis, then resubmit it with the new information. \ No newline at end of file +Song-assigned IDs (analysis and object IDs) and those specified in the [base schema](https://github.com/overture-stack/SONG/blob/develop/song-server/src/main/resources/schemas/analysis/analysisBase.json) are immutable and cannot be altered. If you need to change any of these values, it is recommended to UNPUBLISH and SUPPRESS the analysis, then resubmit it with the new information. \ No newline at end of file diff --git a/docs/02-Usage/03-publication-controls.md b/docs/02-Reference/03-publication-controls.md similarity index 91% rename from docs/02-Usage/03-publication-controls.md rename to docs/02-Reference/03-publication-controls.md index 970913761..713595e1d 100644 --- a/docs/02-Usage/03-publication-controls.md +++ b/docs/02-Reference/03-publication-controls.md @@ -2,7 +2,7 @@ Managing data release with publication controls. -Administrators control analysis availability to downstream services like such as our indexing service [Maestro](https://docs.overture.bio/docs/core-software/Maestro/overview) across three states: +Administrators control analysis availability to downstream services like such as our indexing service [Maestro](/develop/Maestro/overview) across three states: - **Unpublished:** By default, new data committed to Songs database is in this state as it awaits file upload. Custom configurations can allow Maestro to index this data, making it available to users to see the pending analysis without availble file data to download. @@ -18,7 +18,7 @@ To ensure the safe removal of indexed data, the suggested state transition for s Updating publication statuses can be done through two main methods: the Song Client command line tool and the Swagger UI. The Swagger UI provides a user-friendly interface for interacting with the Song API, while the Song Client allows you to more easily perform data management operations from the command line. -- **Setting up the Song Client:** For detailed instructions on setting up the Song Client, please refer to the section on [Installing the Song Client](https://docs.overture.bio/docs/core-software/Song/setup#song-client-setup). +- **Setting up the Song Client:** For detailed instructions on setting up the Song Client, please refer to the section on [Installing the Song Client](/develop/Song/setup#song-client-setup). - **Accessing the Swagger UI:** To access the Swagger UI, you need to determine the URL based on your Song setup. @@ -70,5 +70,5 @@ docker exec song-client sh -c "sing suppress -a " 3. **Execute:** Click Execute. The Swagger UI will provide your response and detailed descriptions of all potential response codes. :::info Support -For technical support or specific use cases, please don't hesitate to reach out through our relevant [**community support channels**](https://docs.overture.bio/community/support). +For technical support or specific use cases, please don't hesitate to reach out through our [**support page**](/community/support) or our [**discussion forum**](https://github.com/overture-stack/docs/discussions?discussions_q=). ::: diff --git a/docs/02-Usage/04-data-model-management.md b/docs/02-Reference/04-data-model-management.md similarity index 59% rename from docs/02-Usage/04-data-model-management.md rename to docs/02-Reference/04-data-model-management.md index fc43ae865..6b4078547 100644 --- a/docs/02-Usage/04-data-model-management.md +++ b/docs/02-Reference/04-data-model-management.md @@ -23,10 +23,10 @@ When submitting an analysis to Song, you must specify an 'analysis type' in your The schema for each analysis type consists of two components: -1. **Base Schema**: A minimal set of essential fields required for all analyses, including: - - Basic patient data - - Submitter IDs - - File details +1. **Base Schema**: A minimal set of essential fields required for all analyses: + - `studyId` — the study the analysis belongs to + - `analysisType` — the analysis type used to validate the submission + - `files` — the file(s) the analysis describes 2. **Dynamic schema**: A flexible component that Song administrators can configure and upload to define specific analysis types. @@ -37,15 +37,20 @@ This two-part schema structure ensures: ### Base Schema -The **base schema** defines the minimal data set required for a schema. It includes non-identifiable primary keys and basic descriptors for patient and cancer sample data: +The **base schema** defines the minimal data set required for every analysis. It requires only three top-level fields: -- Identifiers: Donor ID, Specimen ID, and Sample ID -- Essential cancer sample characteristics +- `studyId` — identifies the study the analysis belongs to +- `analysisType` — the name (and optional version) of the analysis type used for validation +- `files` — an array describing at least one file, including its data type, name, size, access level, type, and MD5 checksum You can view the current base schema in the [Song repository](https://github.com/overture-stack/SONG/blob/develop/song-server/src/main/resources/schemas/analysis/analysisBase.json). +:::note Base schema change in Song 5.3.0 +Prior to Song 5.3.0, the base schema also required donor, specimen, and sample entities, which were stored across multiple related tables. As of 5.3.0 these are no longer required, and all analysis data is stored in a single consolidated table. Existing deployments must migrate their data — see [**Database Migration**](./11-database-migration.md). +::: + :::info Future Updates to our Submission System -As part of our work on the [Pan-Canadian Genome Library](https://oicr.on.ca/first-ever-national-library-of-genomic-data-will-help-personalize-cancer-treatment-in-canada-and-around-the-world/), we are improving our [**data submission system**](https://docs.overture.bio/docs/under-development/). This system will better support tabular (clinical) data and reduce the constraints of Song's base schema, ultimately enhancing the flexibility and robustness of our data management and storage system. For more information [**see our under development section**](https://docs.overture.bio/docs/under-development/). +As part of our work on the [Pan-Canadian Genome Library](https://oicr.on.ca/first-ever-national-library-of-genomic-data-will-help-personalize-cancer-treatment-in-canada-and-around-the-world/), we are improving our [**data submission system**](/develop/Lyric/overview). This system will better support tabular (clinical) data and reduce the constraints of Song's base schema, ultimately enhancing the flexibility and robustness of our data management and storage system. For more information [**see the Lyric documentation**](/develop/Lyric/overview). ::: ### Dynamic schema @@ -74,6 +79,96 @@ The basic portion of a dynamic schema requires at a minimum: For a detailed guide on building JSON Schemas for Song see our [**administration guide on updating data models**](/guides/administration-guides/updating-the-data-model) ::: +## Schema Options + +The `options` property defines extra validations for an analysis schema, such as restrictions on file types and checks against an external service. The `options` property is not required, and each property within it is also optional. If no value is provided for an `options` property, a default configuration is used for the analysis. When updating an existing analysis type, you can omit any option and its value is maintained from the previous version. + +```json +{ + "options": { + "fileTypes": ["bam", "cram"], + "externalValidations": [ + { + "url": "http://localhost:8099/", + "jsonPath": "experiment.someId" + } + ] + } +} +``` + +To remove the previous value of an option so that its validation is no longer required — for instance, removing the restriction on file types so that any file type is allowed — provide an empty list for that option. In the example below, both `fileTypes` and `externalValidations` are set to empty arrays, so these validations are not applied to submitted analyses: + +```json +{ + "options": { + "fileTypes": [], + "externalValidations": [] + } +} +``` + +### File Types + +`options.fileTypes` accepts an array of strings representing the file types (file extensions) allowed for this type of analysis. + +If an empty array is provided, any file type is allowed. If an array of file types is provided, an analysis is invalid if it contains files of a type not listed. + +```json +{ + "options": { + "fileTypes": ["bam", "cram"] + } +} +``` + +### External Validation + +External validations configure Song to check a value in the analysis against an external service by sending an HTTP GET request to a configurable URL. The service should respond with a `2XX` status to indicate the value is valid. + +For example, if a project's clinical data is managed in a separate service, you can add an external validation on the donor ID field of your custom schema. This sends the donor ID to the external service, which can confirm that the donor was previously registered: + +```json +{ + "url": "http://example.com/{study}/donor/{value}", + "jsonPath": "experiment.donorId" +} +``` + +The URL is a template with two variables that are replaced during validation. Song replaces the `{value}` token with the value read from the analysis at the property defined by `jsonPath`, and replaces the `{study}` token with the study ID for the analysis. + +Continuing the example above, if the following analysis was submitted: + +```json +{ + "studyId": "ABC123", + "analysisType": { + "name": "minimalExample" + }, + "files": [ + { + "dataType": "text", + "fileName": "file1.txt", + "fileSize": 123, + "fileType": "txt", + "fileAccess": "open", + "fileMd5sum": "595f44fec1e92a71d3e9e77456ba80d1" + } + ], + "experiment": { + "donorId": "id01" + } +} +``` + +Song would validate the `donorId` by sending a request to `http://example.com/ABC123/donor/id01`. + +The URL parsing allows using either the `{study}` or `{value}` placeholder multiple times (for example, `http://example.com/{study}-{value}/{value}`); each instance is interpolated accordingly. + +:::warning +The URL may cause errors in Song if it contains any tokens matching the `{word}` format other than `{study}` and `{value}`. +::: + ## Registering Analysis Types These steps apply both for registering new schemas and updating existing ones. @@ -83,7 +178,7 @@ These steps apply both for registering new schemas and updating existing ones. 1. **Locate the Endpoint** - From the schema dropdown, find the `POST` **RegisterAnalysisType** endpoint. - ![Register new schema](../assets/swagger_register_schema(s).png 'Register new schema') + ![Register new schema](../assets/swagger_register_schemas.png 'Register new schema') 2. **Input Your Data** - Click *Try it out* & enter your authorization token in the authorization field @@ -216,5 +311,5 @@ curl --location --request GET 'https://song-url.example.com/schemas/sequencing_e ``` :::info Support -For technical support or specific use cases, please don't hesitate to reach out through our relevant [**community support channels**](https://docs.overture.bio/community/support). +For technical support or specific use cases, please don't hesitate to reach out through our [**support page**](/community/support) or our [**discussion forum**](https://github.com/overture-stack/docs/discussions?discussions_q=). ::: diff --git a/docs/02-Usage/05-schema-strictness.md b/docs/02-Reference/05-schema-strictness.md similarity index 89% rename from docs/02-Usage/05-schema-strictness.md rename to docs/02-Reference/05-schema-strictness.md index 963d6b0c5..694e620a1 100644 --- a/docs/02-Usage/05-schema-strictness.md +++ b/docs/02-Reference/05-schema-strictness.md @@ -32,5 +32,5 @@ SCHEMAS_ENFORCELATEST=true By setting `SCHEMAS_ENFORCELATEST` to true, the Song server will enforce that data conforms to the latest schema versions. Conversely, if set to false, data can be submitted under any schema version. :::info Need Help? -If you encounter any issues or have questions about our API, please don't hesitate to reach out through our relevant [**community support channels**](https://docs.overture.bio/community/support). +If you encounter any issues or have questions about our API, please don't hesitate to reach out through our [**support page**](/community/support) or our [**discussion forum**](https://github.com/overture-stack/docs/discussions?discussions_q=). ::: diff --git a/docs/02-Reference/06-id-management.md b/docs/02-Reference/06-id-management.md new file mode 100644 index 000000000..5297918ea --- /dev/null +++ b/docs/02-Reference/06-id-management.md @@ -0,0 +1,23 @@ +# ID Management + +Song assigns unique identifiers to the two entities it tracks: **analyses** and **files**. Both are generated internally by Song in UUID format — no external ID service is required. + +## Analysis IDs + +Each analysis is assigned an **Analysis ID**, a randomly generated UUID, when it is created. Song registers each Analysis ID to guarantee uniqueness across the system. Analysis IDs are used to track, retrieve, and manage an analysis throughout Song and Score. + +## File (Object) IDs + +Each file within an analysis is assigned a **File ID** (also referred to as an object ID). File IDs are deterministic: Song computes them as a name-based (UUID5, SHA-1) hash of the analysis ID and file name. Because they are derived from these inputs rather than stored to enforce uniqueness, the same analysis ID and file name always resolve to the same File ID. + +## Immutability + +Song-assigned IDs are immutable and cannot be altered once created. If you need to change a value that contributes to an ID, UNPUBLISH and SUPPRESS the analysis, then resubmit it with the new information. See [**Updating Metadata**](./02-updating-metadata.md) for details. + +:::note ID management change in Song 5.3.0 +Earlier versions of Song also managed donor, specimen, and sample IDs and supported a "federated" mode that delegated ID generation to an external ID service. As of 5.3.0, donor, specimen, and sample entities have been removed from the base schema, and Song generates the remaining analysis and file IDs internally. For upgrading existing deployments, see [**Database Migration**](./11-database-migration.md). +::: + +:::info Need Help? +If you encounter any issues or have questions about our API, please don't hesitate to reach out through our [**support page**](/community/support) or our [**discussion forum**](https://github.com/overture-stack/docs/discussions?discussions_q=). +::: diff --git a/docs/02-Usage/07-kafka.md b/docs/02-Reference/07-kafka.md similarity index 87% rename from docs/02-Usage/07-kafka.md rename to docs/02-Reference/07-kafka.md index 55a5c5a56..3aa025d4d 100644 --- a/docs/02-Usage/07-kafka.md +++ b/docs/02-Reference/07-kafka.md @@ -18,7 +18,7 @@ This automated flow ensures Song publications are indexed. To enable Kafka integration, you need to activate the `kafka` profile in your Song deployment. This can be done by setting the following environment variable: ```bash -SPRING_CONFIG_ACTIVATE_ON_PROFILE=kafka +SPRING_PROFILES_ACTIVE=kafka ``` ### Basic Configuration @@ -46,11 +46,11 @@ Here's an example of how to set these variables in a Docker environment file: ```bash # Kafka Configuration -SPRING_CONFIG_ACTIVATE_ON_PROFILE=kafka +SPRING_PROFILES_ACTIVE=kafka SPRING_KAFKA_BOOTSTRAP_SERVERS=kafka:9092 SPRING_KAFKA_TEMPLATE_DEFAULT_TOPIC=song-analysis ``` :::info Need Help? -If you encounter any issues or have questions about our API, please don't hesitate to reach out through our relevant [**community support channels**](https://docs.overture.bio/community/support). +If you encounter any issues or have questions about our API, please don't hesitate to reach out through our [**support page**](/community/support) or our [**discussion forum**](https://github.com/overture-stack/docs/discussions?discussions_q=). ::: \ No newline at end of file diff --git a/docs/02-Usage/08-authentication.md b/docs/02-Reference/08-authentication.md similarity index 89% rename from docs/02-Usage/08-authentication.md rename to docs/02-Reference/08-authentication.md index f3aaac8eb..2846dd8b0 100644 --- a/docs/02-Usage/08-authentication.md +++ b/docs/02-Reference/08-authentication.md @@ -164,7 +164,7 @@ Update your Song configuration by adding these Keycloak variables to your `.env. ```bash # Keycloak Integration -SPRING_CONFIG_ACTIVATE_ON_PROFILE=secure +SPRING_PROFILES_ACTIVE=secure # Server Configuration AUTH_SERVER_PROVIDER=keycloak @@ -192,17 +192,16 @@ Replace any default values with the values specific to your environment. The var **Server Authentication Integration** - `AUTH_SERVER_PROVIDER`: Required - Specify the authentication server provider. In this case, it's set to `keycloak` - `AUTH_SERVER_KEYCLOAK_HOST`: Required - The host address for the Keycloak server. Default is `http://localhost` update this variable accordingly -- `AUTH_SERVER_KEYCLOAK_REALM`: Required - The realm in Keycloak under which the Score service is registered. Example: `myrealm` -- `AUTH_SERVER_URL`: Required - URL for the Keycloak API endpoint authenticating a user's API key. Specify the full endpoint URL by inserting your realm name +- `AUTH_SERVER_KEYCLOAK_REALM`: Required - The realm in Keycloak under which the Song service is registered. Example: `myrealm` +- `AUTH_SERVER_INTROSPECTIONURI`: Required - URL for the Keycloak API endpoint that validates a user's API key. Specify the full endpoint URL by inserting your realm name - `AUTH_SERVER_TOKENNAME`: Required - Name identifying a token. Keep this as the default value `apiKey` -- `AUTH_SERVER_CLIENTID`: Required - The client ID for the Score application configured in Keycloak -- `AUTH_SERVER_CLIENTSECRET`: Required - The client secret for the Score application configured in Keycloak. This can be accessed from the **"Client details"** under the **"Credentials tab"** +- `AUTH_SERVER_CLIENTID`: Required - The client ID for the Song application configured in Keycloak +- `AUTH_SERVER_CLIENTSECRET`: Required - The client secret for the Song application configured in Keycloak. This can be accessed from the **"Client details"** under the **"Credentials tab"** **Scope Configuration** -- `AUTH_SERVER_SCOPE_DOWNLOAD_SYSTEM`: Required - Scope (permission) for system-level downloads from Score using an API key. Default: `score.WRITE` -- `AUTH_SERVER_SCOPE_DOWNLOAD_SUFFIX`: Required - Suffix after the Song study name when assigning study-level download scopes for Score. Default: `.READ` -- `AUTH_SERVER_SCOPE_UPLOAD_SYSTEM`: Required - Scope (permission) for system-level uploads to Score using an API key. If following the above instructions for application setup this value will be `score-api.` -- `AUTH_SERVER_SCOPE_UPLOAD_SUFFIX`: Required - Suffix after the Song study name when assigning study-level upload scopes for Score. Default: `.WRITE` +- `AUTH_SERVER_SCOPE_STUDY_PREFIX`: Required - Prefix applied before the Song study name when assigning study-level scopes. Default: `STUDY.` +- `AUTH_SERVER_SCOPE_STUDY_SUFFIX`: Required - Suffix applied after the Song study name when assigning study-level scopes. Default: `.WRITE` +- `AUTH_SERVER_SCOPE_SYSTEM`: Required - Scope (permission) for system-level access to Song using an API key. Default: `song.WRITE` **JWT Configuration** - `SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_JWKSETURI`: Required - URI for JWT JSON Web Key Set (JWK Set) for the OAuth2 resource server. Specify the Keycloak server URI by inserting your realm name @@ -210,5 +209,5 @@ Replace any default values with the values specific to your environment. The var
:::info Need Help? -If you encounter any issues or have questions about our API, please don't hesitate to reach out through our relevant [**community support channels**](https://docs.overture.bio/community/support). +If you encounter any issues or have questions about our API, please don't hesitate to reach out through our [**support page**](/community/support) or our [**discussion forum**](https://github.com/overture-stack/docs/discussions?discussions_q=). ::: diff --git a/docs/02-Usage/09-api-reference.mdx b/docs/02-Reference/09-api-reference.mdx similarity index 85% rename from docs/02-Usage/09-api-reference.mdx rename to docs/02-Reference/09-api-reference.mdx index 82c45dbbe..3d506639b 100644 --- a/docs/02-Usage/09-api-reference.mdx +++ b/docs/02-Reference/09-api-reference.mdx @@ -6,7 +6,7 @@ Songs API is a RESTful API that uses JSON for request and response bodies. It fo Depending on your environment, the Songs Swagger API can be accessed from the following URLs: -- Development URL: `http://localhost:8080/swagger.html` +- Development URL: `http://localhost:8080/swagger-ui.html` - Example production URL: `https://song.demo.overture.bio/swagger-ui.html` ### Authentication @@ -35,4 +35,4 @@ import SwaggerAPIDoc from '/src/components/SwaggerAPIDoc'; ### Need Help? -If you encounter any issues or have questions about our API, please don't hesitate to reach out through our relevant [community support channels](https://docs.overture.bio/community/support) \ No newline at end of file +If you encounter any issues or have questions about our API, please don't hesitate to reach out through our [**support page**](/community/support) or our [**discussion forum**](https://github.com/overture-stack/docs/discussions?discussions_q=). \ No newline at end of file diff --git a/docs/02-Usage/10-client-reference.md b/docs/02-Reference/10-client-reference.md similarity index 78% rename from docs/02-Usage/10-client-reference.md rename to docs/02-Reference/10-client-reference.md index f6ca7eefb..94f066afc 100644 --- a/docs/02-Usage/10-client-reference.md +++ b/docs/02-Reference/10-client-reference.md @@ -8,13 +8,13 @@ Commands and options supported by the Song client. - The `config` command shows the current configuration settings. -- **Usage:** `song-client config` +- **Usage:** `sing config` ### Ping - The `ping` command can test the connection to the Song server. -- **Usage:** `song-client ping` +- **Usage:** `sing ping` ## Analysis Management Commands @@ -22,7 +22,7 @@ Commands and options supported by the Song client. - Retrieves specific analysis type schema information. -- **Usage:** `song-client get-analysis-type [OPTIONS]` +- **Usage:** `sing get-analysis-type [OPTIONS]` | Option | Description | |--------|-------------| @@ -34,7 +34,7 @@ Commands and options supported by the Song client. - Lists all analysis types with filtering and viewing options. -- **Usage:** `song-client list-analysis-types [OPTIONS]` +- **Usage:** `sing list-analysis-types [OPTIONS]` | Option | Description | |--------|-------------| @@ -51,7 +51,7 @@ Commands and options supported by the Song client. - Registers a new analysis-type schema. -- **Usage:** `song-client register-analysis-type [OPTIONS]` +- **Usage:** `sing register-analysis-type [OPTIONS]` | Option | Description | |--------|-------------| @@ -61,7 +61,7 @@ Commands and options supported by the Song client. - Submits a payload to create an analysis. -- **Usage:** `song-client submit [OPTIONS]` +- **Usage:** `sing submit [OPTIONS]` | Option | Description | |-----------------------------|-----------------------------------------------------| @@ -69,28 +69,25 @@ Commands and options supported by the Song client. | `-ad`, `--allow-duplicates` | Allows duplicate files identified by their MD5 hash | :::info - For detailed information, see our [documentation on submitting data with Song](https://docs.overture.bio/docs/core-software/Song/Usage/submitting-metadata). + For detailed information, see our [documentation on submitting data with Song](/develop/Song/Reference/submitting-metadata). ::: ### Search - Searches for analysis objects based on various parameters. -- **Usage:** `song-client search [OPTIONS]` +- **Usage:** `sing search [OPTIONS]` | Option | Description | |--------|-------------| | `-a`, `--analysis-id` | Search by analysisId | - | `-d`, `--donor-id` | Search by donorId | | `-f`, `--file-id` | Search by fileId | - | `-sa`, `--sample-id` | Search by sampleId | - | `-sp`, `--specimen-id` | Search by specimenId | ### Manifest - Generates a manifest file for an analysis. -- **Usage:** `song-client manifest [OPTIONS]` +- **Usage:** `sing manifest [OPTIONS]` | Option | Description | |--------|-------------| @@ -99,14 +96,14 @@ Commands and options supported by the Song client. | `-d`, `--input-dir` | Directory containing upload files | :::info - For more information, see our [documentation on submitting data with Song](https://docs.overture.bio/docs/core-software/Song/Usage/submitting-metadata). + For more information, see our [documentation on submitting data with Song](/develop/Song/Reference/submitting-metadata). ::: ### Publish - Publishes an analysis. -- **Usage:** `song-client publish [OPTIONS]` +- **Usage:** `sing publish [OPTIONS]` | Option | Description | |--------|-------------| @@ -117,7 +114,7 @@ Commands and options supported by the Song client. - Marks data as unavailable to downstream services. -- **Usage:** `song-client unpublish [OPTIONS]` +- **Usage:** `sing unpublish [OPTIONS]` | Option | Description | |--------|-------------| @@ -127,7 +124,7 @@ Commands and options supported by the Song client. - Blocks data from being accessed. -- **Usage:** `song-client suppress [OPTIONS]` +- **Usage:** `sing suppress [OPTIONS]` | Option | Description | |--------|-------------| @@ -135,14 +132,14 @@ Commands and options supported by the Song client. :::info - For more information on analysis management, see our [documentation on Song publication controls](https://docs.overture.bio/docs/core-software/Song/Usage/publication-controls). + For more information on analysis management, see our [documentation on Song publication controls](/develop/Song/Reference/publication-controls). ::: ### Export - Exports payloads based on various parameters. -- **Usage:** `song-client export [OPTIONS]` +- **Usage:** `sing export [OPTIONS]` | Option | Description | |--------|-------------| @@ -156,7 +153,7 @@ Commands and options supported by the Song client. - Updates file metadata. -- **Usage:** `song-client update-file [OPTIONS]` +- **Usage:** `sing update-file [OPTIONS]` | Option | Description | |--------|-------------| @@ -169,4 +166,4 @@ Commands and options supported by the Song client. ## Need Help? -If you encounter any issues or have questions, please don't hesitate to reach out through our relevant [community support channels](https://docs.overture.bio/community/support) \ No newline at end of file +If you encounter any issues or have questions, please don't hesitate to reach out through our [**support page**](/community/support) or our [**discussion forum**](https://github.com/overture-stack/docs/discussions?discussions_q=). \ No newline at end of file diff --git a/docs/02-Reference/11-database-migration.md b/docs/02-Reference/11-database-migration.md new file mode 100644 index 000000000..554b99be1 --- /dev/null +++ b/docs/02-Reference/11-database-migration.md @@ -0,0 +1,67 @@ +# Database Migration + +Migrating an existing Song database to the consolidated schema introduced in Song 5.3.0. + +## Overview + +Song 5.3.0 introduces a significant internal change to the database schema. In earlier versions, data related to donor, specimen, and sample entities was stored across multiple tables, each with required relationships enforced by the [base schema](./04-data-model-management.md). In 5.3.0 those requirements have been removed in favour of a simpler, more flexible model: all analysis data is now stored in a single consolidated table. + +Starting with 5.3.0, all new analysis data is written to this consolidated table. To keep data created by earlier versions compatible with the new structure, existing data must be migrated. + +A TypeScript-based migration script handles this. It reads from and writes to a configured database, with support for pagination and connection pooling, and is designed to run in a Node.js environment (version 20 or higher). + +:::warning Back up your database first +It is best practice to take a full backup of your database before running the migration script. +::: + +## Prerequisites + +- Node.js v20 or higher +- The [pnpm](https://pnpm.io/) package manager +- A database accessible from your environment, with a user that has sufficient privileges to read from and write to the necessary tables + +## Installation + +Clone the migration repository and install its dependencies: + +```bash +git clone https://github.com/overture-stack/song_5_3_migration.git +cd song_5_3_migration +pnpm install +``` + +## Configuration + +Copy `.env.schema`, rename the copy to `.env`, and place it in the root of the project. Adjust the values to match your setup: + +```bash +# Number of analyses fetched per query (pagination) +ANALYSIS_QUERY_LIMIT=100 + +# Required DB configuration +DB_HOST= # Database server host +DB_PORT= # Database server port +DB_NAME= # Name of the target database +DB_USER= # Database username +DB_PASSWORD= # Database user password + +# Max concurrent DB connections in the pool +MAX_DB_CONNECTIONS=10 +``` + +- `ANALYSIS_QUERY_LIMIT` controls how many analyses are fetched per query during pagination. +- `MAX_DB_CONNECTIONS` controls the number of concurrent connections in the pool. Tune it to balance migration speed against database load. + +## Running the migration + +With your `.env` in place, run the script: + +```bash +pnpm run dev +``` + +This transpiles the TypeScript code and executes the main migration entry point using your configured environment, reading existing analyses and rewriting them into the consolidated table. + +:::info Need Help? +If you encounter any issues or have questions about this migration, please don't hesitate to reach out through our [**support page**](/community/support) or our [**discussion forum**](https://github.com/overture-stack/docs/discussions?discussions_q=). +::: diff --git a/docs/02-Usage/06-id-management.md b/docs/02-Usage/06-id-management.md deleted file mode 100644 index 8a99a36b0..000000000 --- a/docs/02-Usage/06-id-management.md +++ /dev/null @@ -1,184 +0,0 @@ -# ID Management - -Song provides two distinct approaches for managing primary keys across entities (Donors, Specimens, Samples, and Files): - -### Local Mode -- Internally manages IDs within Song's system -- Uses UUID format -- Ensures thread safety and consistency through internal memory -- All IDs except analysisId are stateless and computed using UUID5 hash -- AnalysisIDs are stateful to guarantee uniqueness - -### Federated Mode -- Relies on external service for ID management -- Supports two authentication methods: - - Dynamic JWT tokens - - Predefined static keys -- Requires validation against external ID database -- Stores validated IDs in Song database - -:::info Important -You must choose either Local or Federated mode for all IDs. Mixing modes is not supported. -::: - -## Configuration - -### Local Mode Setup - -To use Song's internal ID management system: - -```env -ID_USELOCAL=true -``` - -:::info - The `AnalysisService.create` method handles analysisId registration, so `LocalIdService` doesn't need to save/register analysisIds directly. -::: - -### Federated Mode Setup - -To use external ID management: - -1. Set the base configuration: -```env -ID_USELOCAL=false -``` - -2. Configure entity-specific URI templates: -```env -# Entity URIs -ID_FEDERATED_URITEMPLATE_DONOR=https://id.server.org/donor/id?projectCode={studyId}&donorSubmittedId={submitterId}&create=true -ID_FEDERATED_URITEMPLATE_SPECIMEN=https://id.server.org/specimen/id?projectCode={studyId}&specimenSubmittedId={submitterId}&create=true -ID_FEDERATED_URITEMPLATE_SAMPLE=https://id.server.org/sample/id?projectCode={studyId}&sampleSubmittedId={submitterId}&create=true -ID_FEDERATED_URITEMPLATE_FILE=https://id.server.org/file/id?bundleId={analysisId}&fname={fileName} - -# Analysis-specific URIs -ID_FEDERATED_URITEMPLATE_ANALYSIS_EXISTENCE=https://id.server.org/analysis/id?submittedAnalysisId={analysisId}&create=false -ID_FEDERATED_URITEMPLATE_ANALYSIS_GENERATE=https://id.server.org/analysis/id/generate -ID_FEDERATED_URITEMPLATE_ANALYSIS_SAVE=https://id.server.org/analysis/id?submittedAnalysisId={submitterId}&create=true -``` - -3. Configure authentication: -```env -# Base auth URL -ID_FEDERATED_AUTH_URL=https://auth.server.org - -# For static authentication (FEDERATED_STATIC_AUTH) -ID_FEDERATED_AUTH_BEARER_TOKEN=your_static_token - -# For dynamic authentication (FEDERATED_DYNAMIC_AUTH) -ID_FEDERATED_AUTH_BEARER_CREDENTIALS_CLIENTID=authClientID -ID_FEDERATED_AUTH_BEARER_CREDENTIALS_CLIENTSECRET=authClientSecret -``` - -### Application YAML Configuration - -You can configure ID Management in your `application.yaml` file. Here are the available options: - -```yaml -id: - # Enable local ID management - useLocal: true # Set to false for federated mode - - # Optional: Enable in-memory persistence for testing - persistInMemory: true # Only recommended for development/testing - - # Federated mode configuration - federated: - # Authentication configuration - auth: - bearer: - # Static token authentication - token: "your-static-token" - # Dynamic authentication credentials - credentials: - url: "https://auth.server.org" - clientId: "your-client-id" - clientSecret: "your-client-secret" - - # URI templates for federated services - uriTemplate: - # Entity ID endpoints - donor: "https://id.example.org/donor/id?submittedProjectId={studyId}&submittedDonorId={submitterId}&create=true" - specimen: "https://id.example.org/specimen/id?submittedProjectId={studyId}&submittedSpecimenId={submitterId}&create=true" - sample: "https://id.example.org/sample/id?submittedProjectId={studyId}&submittedSampleId={submitterId}&create=true" - file: "https://id.example.org/file/id?bundleId={analysisId}&fname={fileName}" - - # Analysis-specific endpoints - analysisExistence: "https://id.example.org/analysis/id?submittedAnalysisId={analysisId}&create=false" - analysisGenerate: "https://id.example.org/analysis/id/generate" - analysisSave: "https://id.example.org/analysis/id?submittedAnalysisId={submitterId}&create=true" -``` - -### Profile-Specific Configuration - -Song supports different configuration profiles. Here's how to configure ID management for specific profiles: - -```yaml ---- -spring: - config: - activate: - on-profile: dev -id: - persistInMemory: true # Enable in-memory persistence for development - ---- -spring: - config: - activate: - on-profile: prod -id: - useLocal: false # Use federated mode in production - federated: - auth: - bearer: - credentials: - url: "https://prod-auth.example.org" - clientId: ${PROD_CLIENT_ID} - clientSecret: ${PROD_CLIENT_SECRET} -``` - - -## URI Requirements - -When using federated mode, the external ID service must: - -- Implement GET controllers for all configured URI templates -- Support either static or dynamic authentication -- Return appropriate responses for each entity type - -### Required URI Parameters - -| Entity Type | Required Variables | Example URI | Response Type | -|-------------|-------------------|-------------|---------------| -| Donor | studyId, submitterId | `/donor/id?projectCode={studyId}&donorSubmittedId={submitterId}` | plaintext | -| Specimen | studyId, submitterId | `/specimen/id?projectCode={studyId}&specimenSubmittedId={submitterId}` | plaintext | -| Sample | studyId, submitterId | `/sample/id?projectCode={studyId}&sampleSubmittedId={submitterId}` | plaintext | -| File | analysisId, fileName | `/file/id?bundleId={analysisId}&fname={fileName}` | plaintext | -| Analysis Existence | analysisId | `/analysis/id?submittedAnalysisId={analysisId}&create=false` | plaintext | -| Analysis Generate | none | `/analysis/id/generate` | plaintext | -| Analysis Save | analysisId | `/analysis/id?submittedAnalysisId={submitterId}&create=true` | none | - -### ICGC ARGO Example - -The ICGC ARGO Data Platform is an international initiative with several distributed processing centres. This required the use of a central ID Service. An example of a URI donor request used by this system is as follows: - -`https://clinical.platform.icgc-argo.org/clinical/donors/id?programId=PACA-CA&submitterId=PCSI_0591` - -In the provided URI, a researcher requests the centralized ID service to retrieve the unique identifier for a **donor** associated with the programId **PACA-CA** and the submitterID **PCSI_0591**. - -#### 200 Response: - -```shell -DO224719 -``` - -#### 404 Response: - -```json -{ - "error": "Error", - "message": "Donor not found" -} -``` diff --git a/docs/assets/swagger_register_schema(s).png b/docs/assets/swagger_register_schemas.png similarity index 100% rename from docs/assets/swagger_register_schema(s).png rename to docs/assets/swagger_register_schemas.png diff --git a/docs/custom-schemas.md b/docs/custom-schemas.md deleted file mode 100644 index ef992a4cc..000000000 --- a/docs/custom-schemas.md +++ /dev/null @@ -1,110 +0,0 @@ -# Custom Analysis Schemas - -## Minimal Example - -```json -{ - "name": "minimalExample", - "options":{}, - "schema":{ - "type": "object", - "required":[ - "experiment" - ], - "properties":{ - "experiment":{} - } - } -} -``` - -## Options - -The `options` property defines extra validations for this analysis schema, such as restrictions on file types and checks on the data with an external service. The `options` property is not required. Similarly, each property in `options` is also optional. If no value is provided for one of the `options` properties, a default configuration will be used for the analysis. If this is an update to an existing analysis type, you can omit any option and its value will be maintained from the previous version. - -```json -{ - "fileTypes":["bam", "cram"], - "externalValidations":[ - { - "url": "http://localhost:8099/", - "jsonPath": "experiment.someId" - } - ] -}, -``` - -If you want to remove the previous value of an option so that this validation is no longer required, for instance removing the restriction on file types so that any file type could be provided, you should provide an empty list for that option. - -In the example below, both `fileTypes` and `externalValidations` properties are set to empty arrays, which means that these validations will not be applied to submitted analysis: - -```json -{ - "options": { - "fileTypes": [], - "externalValidations": [] - } -} -``` - -### File Types - -`options.fileTypes` can be provided an array of strings. These represent the file types (file extensions) that are allowed to be uploaded for this type of analysis. - -If an empty array is provided, then any file type will be allowed. If an array of file types is provided, then an analysis will be invalid if it contains files of a type not listed. - -```json -{ - "options": { - "fileTypes": ["bam","cram"] - } -} -``` - -### External Validation - -External validations configure Song to check a value in the analysis against an external service by sending an HTTP GET request to a configurable URL. The service should respond with a 2XX status message to indicate the value "is valid". - -As an example, if the project clinical data is being managed in a separate service, we can add an external validation to the donor id field of our custom scheme. This will send the donor id to the external service which can confirm that we have previously registered that donor. - -This might look like the following: - -```json -{ - "url": "http://example.com/{study}/donor/{value}", - "jsonPath": "experiment.donorId" -} -``` - -The URL provided is a template, with two variables that will be replaced during validation. Song will replace the token `{value}` with the value read from the analysis at the property as defined in the `jsonPath`. Song will also replace the token `{study}` with the study ID for the Analysis. - -Continuing the above example, if the following analysis was submitted: - -```json -{ - "studyId": "ABC123", - "analysisType": { - "name": "minimalExample" - }, - "files": [ - { - "dataType": "text", - "fileName": "file1.txt", - "fileSize": 123, - "fileType": "txt", - "fileAccess": "open", - "fileMd5sum": "595f44fec1e92a71d3e9e77456ba80d1" - } - ], - "experiment": { - "donorId": "id01" - } -} -``` - -Song would attempt to validate the donorId by sending a validation request to `http://example.com/ABC123/donor/id01`. - -The URL parsing allows using either the `{study}` or `{value}` placeholders multiple times (e.g. `http://example.com/{study}-{value}/{value}`), each instance will be interpolated accordingly. - -> [!Warning] -> The URL may cause errors in Song if it contains any tokens matching the `{word}` format other than `{study}` and `{value}`