diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f25e9b8c..04689fe1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,34 +1,7 @@ variables: - IMAGE: crest-service - VERSION: '0.2' - REGISTRY: 'svomtest.svom.fr:5543' -# REGISTRY: 'svomtest.svom.fr:5543' - CONTAINER_IMAGE: $REGISTRY/$IMAGE:$VERSION - WEB_CONTAINER_IMAGE: $REGISTRY/crest-ui:$VERSION + IMAGE: crest21-service + VERSION: '6.2' GRADLE_OPTS: "-Dorg.gradle.daemon=false" - GIT_SSL_NO_VERIFY: "true" - - # When using dind service we need to instruct docker, to talk with the - # daemon started inside of the service. The daemon is available with - # a network connection instead of the default /var/run/docker.sock socket. - # - # The 'docker' hostname is the alias of the service container as described at - # https://docs.gitlab.com/ee/ci/docker/using_docker_images.html#accessing-the-services - # - # Note that if you're using Kubernetes executor, the variable should be set to - # tcp://localhost:2375 because of how Kubernetes executor connects services - # to the job container -# DOCKER_HOST: tcp://docker:2375/ - # When using dind, it's wise to use the overlayfs driver for - # improved performance. -# DOCKER_DRIVER: overlay2 - -# Select a runner -# use tag keyword (see below) - -#services: -# - docker:dind - # Make the gradle wrapper executable. This essentially downloads a copy of # Gradle to build the project with. @@ -37,168 +10,61 @@ variables: before_script: - chmod +x gradlew -# We redirect the gradle user home using -g so that it caches the -# wrapper and dependencies. -# https://docs.gradle.org/current/userguide/gradle_command_line.html -# -# Unfortunately it also caches the build output so -# cleaning removes reminants of any cached builds. -# The assemble task actually builds the project. -# If it fails here, the tests can't run. -#build: -# stage: build -# script: -# - ./gradlew -g /cache/.gradle clean :crestdb-web:assemble -PwarName=crest.war -# allow_failure: false - -# Use the generated build output to run the tests. -#test: -# stage: test -# script: -# - ./gradlew -g /cache/.gradle check +include: + - project : 'ci-tools/container-image-ci-templates' + file : 'kaniko-image.gitlab-ci.yml' + ref: master + - project : 'atlas-devops/ci-templates' + file : 'helm/helm.yml' + ref: main stages: - package_application - - webui - - build - - release + #- build - sonar_application - - redeploy - -package_svom_application: - stage: package_application - image: openjdk:11-jdk-alpine - script: - - ./gradlew -g /cache/.gradle clean assemble -PwarName=crest.war - allow_failure: false - artifacts: - paths: - - ./crestdb-web/build/libs/crest.war - expire_in: 1 week - only: - - cms-v0 - -release: - stage: release - image: docker:19.03.1 - services: - - docker:19.03.1-dind - before_script: - - docker login -u svom -p $SVOMREGISTRYPSWD $REGISTRY - script: - - docker build --rm --tag=$CONTAINER_IMAGE --file Dockerfile . - - docker push $CONTAINER_IMAGE - only: - - cms-v0 - -webui: - stage: webui - environment: dc0 - tags: - - dc0-shell - script: - - docker info - - docker build --tag=$WEB_CONTAINER_IMAGE --file ./web-ui/crest-ui/Dockerfile.nginx.svom ./web-ui/crest-ui/ - only: - - svom-dc1 - -sonar_application: - stage: sonar_application - image: openjdk:11-jdk-alpine - script: - - ./gradlew sonarqube -Dsonar.projectKey=crestdb -Dsonar.host.url=http://svom1.lal.in2p3.fr:20090 -Dsonar.login=f50918d8b780181f947ae83b42e78983ffde4bf3 - allow_failure: false - only: - - stable - -# build_svom_docker_image: -# stage: build -# environment: dc0 -# tags: -# - dc0-shell -# image: java:8 -# script: -# - docker info -# - docker build --tag=$CONTAINER_IMAGE --file Dockerfile.svom . -# only: -# - svom-dc1 - -# webui: -# stage: webui -# tags: -# - dc0-shell -# script: -# - docker info -# - docker build --tag=$WEB_CONTAINER_IMAGE ./web-ui/crest-ui/ -# only: -# - svom-dc0 - -# registry-push: -# stage: release -# environment: dc0 -# tags: -# - dc0-shell -# script: -# - docker login -u svom -p $SVOMREGISTRYPSWD $REGISTRY -# - docker push $CONTAINER_IMAGE -# only: -# - svom-dc1 + - push package_cern_application: stage: package_application environment: cern - only: - refs: - - master - variables: - - $REMOTE_NAME == "cern" - image: adoptopenjdk:11-jdk-openj9 + image: registry.cern.ch/docker.io/eclipse-temurin:23-alpine + services: + - registry.cern.ch/docker.io/redis:latest + variables: + REDIS_HOST: redis + REDIS_PORT: 6379 script: - - ./gradlew -g /cache/.gradle clean assemble -PwarName=crest.war + - ./gradlew clean build allow_failure: false artifacts: paths: - - ./crestdb-web/build/libs/crest.war + - ./build/libs/crest.jar -package_cern_web_application: - stage: webui - environment: cern - only: - refs: - - master - variables: - - $REMOTE_NAME == "cern" - image: node:lts-alpine - script: - - ./vuecompile.sh - allow_failure: false - artifacts: - paths: - - ./web-ui/crest-ui/dist +# Can be used to test packaging helm chart, before merging to master for instance +#build_helm: +# extends: .deploy_helm +# stage: build +# variables: +# REGISTRY_CHART_PATH: registry.cern.ch/crest/charts -build_cern_docker_image: - stage: build +build_container_job: + rules: + - if: $DO_DOCKER_IMAGE == "yes" && $CI_PIPELINE_SOURCE != 'merge_request_event' && $SITE == "cern" environment: cern - only: - refs: - - master - variables: - - $REMOTE_NAME == "cern" - tags: - - docker-image-build - script: "echo building $CI_REGISTRY_IMAGE:runner" # No empty scripts are allowed + stage: push + extends: .build_kaniko variables: - TO: $CI_REGISTRY_IMAGE:latest # Where to push resulting image + REGISTRY_IMAGE_PATH : "registry.cern.ch/crest/$IMAGE:$VERSION" + CONTEXT_DIR: "" + PUSH_IMAGE: "true" + DOCKER_FILE_NAME: "Dockerfile" -redeploy: - stage: redeploy - environment: cern - only: - refs: - - master - variables: - - $REMOTE_NAME == "cern" - image: gitlab-registry.cern.ch/paas-tools/openshift-client:latest - script: "oc import-image $IMAGE_NAME --server=$SERVER --namespace $NAMESPACE --all --token=$IMAGE_IMPORT_TOKEN" +push_helm: + extends: .deploy_helm + stage: push + variables: + REGISTRY_CHART_PATH: registry.cern.ch/crest/charts + PUSH_CHART: "true" + diff --git a/.idea/.name b/.idea/.name deleted file mode 100644 index ca306f87..00000000 --- a/.idea/.name +++ /dev/null @@ -1 +0,0 @@ -crestdb \ No newline at end of file diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml deleted file mode 100644 index a55e7a17..00000000 --- a/.idea/codeStyles/codeStyleConfig.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/uiDesigner.xml b/.idea/uiDesigner.xml deleted file mode 100644 index e96534fb..00000000 --- a/.idea/uiDesigner.xml +++ /dev/null @@ -1,124 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.openapi-generator-ignore b/.openapi-generator-ignore new file mode 100644 index 00000000..14374cfe --- /dev/null +++ b/.openapi-generator-ignore @@ -0,0 +1,25 @@ +# OpenAPI Generator Ignore +# Generated by openapi-generator https://github.com/openapitools/openapi-generator + +# Use this file to prevent files from being overwritten by the generator. +# The patterns follow closely to .gitignore or .dockerignore. +src/main/java/** + + +# As an example, the C# client generator defines ApiClient.cs. +# You can make changes and tell OpenAPI Generator to ignore just this file by uncommenting the following line: +#ApiClient.cs + +# You can match any string of characters against a directory, file or extension with a single asterisk (*): +#foo/*/qux +# The above matches foo/bar/qux and foo/baz/qux, but not foo/bar/baz/qux + +# You can recursively match patterns against a directory, file or extension with a double asterisk (**): +#foo/**/qux +# This matches foo/bar/qux, foo/baz/qux, and foo/bar/baz/qux + +# You can also negate patterns with an exclamation (!). +# For example, you can ignore all files in a docs folder with the file extension .md: +#docs/*.md +# Then explicitly reverse the ignore rule for a single file: +#!docs/README.md diff --git a/.project b/.project index 04cc93b9..53909bd5 100644 --- a/.project +++ b/.project @@ -20,4 +20,15 @@ org.eclipse.jdt.core.javanature org.eclipse.buildship.core.gradleprojectnature + + + 1683893511389 + + 30 + + org.eclipse.core.resources.regexFilterMatcher + node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ + + + diff --git a/CREST_DOC.md b/CREST_DOC.md new file mode 100644 index 00000000..7ea6851a --- /dev/null +++ b/CREST_DOC.md @@ -0,0 +1,109 @@ +![Java CI with Gradle](https://github.com/HSF/Crest/workflows/Java%20CI%20with%20Gradle/badge.svg?event=push) + +#### Author: A.Formica, R.Sipos +#### Contributors: M.Mineev, E.Alexandrov (client tools) +``` + Copyright (C) 2016 A.Formica, R.Sipos + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +``` +# Table of Contents +1. [Description](#description) +2. [Data Model](#data-model-overview) +1. [Workflows](#workflows) + +## Description +CREST service is a RESTful API for the storage and retrieval of IOVs (Interval of Validity) and payloads. +The data model is illustrated in the following diagram: + +```mermaid +classDiagram + GlobalTag --* Maps + Tag --* Maps + Tag --* Iov + Iov *--|> Payload + class GlobalTag{ + +String name + +String workflow + } + class Tag{ + +String name + +Long endTime + } + class Iov{ + +Long since + +String tagName + +Timestamp insertionTime + +String payloadHash + } + class Maps{ + +String globalTagName + +String tagName + +String label + +String record + } + class Payload{ + +String hash + +binary data + +String objectType + } +``` + +The entity relationship in the relational DB is represented in the following diagram: + +```mermaid +erDiagram + GlobalTag ||..|{ GlobalTagMaps : maps + GlobalTagMaps }|..|| Tag: maps + Tag ||..|| Meta: description + Tag ||..o{ Iov : links + Iov }|--|| Payload : has + Payload ||..|| Data : contains + Payload ||..|| Streamer : contains +``` +## Data Model overview +1. `TAG` : a tag is a virtual container of metadata (the `IOVs`), and is defined via the following properties + * `name` : a string identifying the tag in a unique way + * `description` : a general description of the tag + * `timeType` : the time type of the tag [time, run-lumi,...] + * `payloadSpec` : the type of the object that the tag is associated with (a CrestContainer can generate either `crest-json-single-iov`, or `crest-json-multi-iov`) + * `synchronization` : the synchronization of the tag, this is still to be defined, but could be `UPDX` for example (optional, can be taken from tag name). + * `lastValidatedTime`: The last validated time of the tag, in millisecon (optional, not used in Athena). + * `endOfValidity` : the end of validity of the tag, in the same units as the "since" of the IOVs (see later). + +2. `TAGMETA` : this object has been introduced mainly to provide a correct mapping with the existing `metadata` of `COOL`, in particular related to `channels` list and `folder` specifications. It is in one-to-one relationship with the `TAG`. It is essential to have these metadata in order to increase compatibility with COOL related code in Athena. We will provide more details below on its content. For Athena usage, it is mandatory to create a TAGMETA entry for each TAG that a user create. + +2. `IOV` : the metadata represeting the start of validity for a given payload. It is defined via the following properties + * `tagName` : the name of the tag. + * `since` : the since of the IOV. + * `insertionTime` : the insertion time of the IOV. Provided by the server. + * `payloadHash` : the unique key for the payload. It is computed as the SHA256 hash of the payload by the server. + In CREST the concept of IOV is always *open-ended*, in the sense that it is valid until the next since. The only IOV for which we foresee a possible *end time* is the last IOV; when the *end time* is not INFINITY, then it can be provided during the PAYLOAD upload. It will update the *endOfValidity* field in the TAG table. Any search functionality will in any case only use the *since* field in CREST, and eventually the *insertionTime*. The latter is used to *go back* into the history of a given *since*, in the case that it has been overridden by the expert with another PAYLOAD. + +4. `PAYLOAD` : it is the binary object containing the conditions data payload. In our default serialization this is a JSON file, created with the help of `CrestContainer`. This allows to transparently migrate existing COOL data in a format that can then be used to generate a generic `CoralAttributeList` container on the Athena side. In terms of server implementation, the CREST server itself and its underlying relational DB tables are completely agnostic as far as the serialization choice of the PAYLOAD is concerned. For obvious reason, that is not at all the case on the client side, so it is important to understand the implication of the chosen serialization, and eventually adopt a strategy which is optimal for the given payload type. + +The CREST API can then be seen as a *KEY=VALUE* store for PAYLOAD files. The TAG identifies a *payload type* (a combination of *folder* and *tag* in COOL), and every TAG will contain the *time history* of the PAYLOAD. When looking at the IOV in a tag, we get a unique *KEY* (the `payloadHash`) corresponding to a given time. Using that *KEY* we can download the corresponding PAYLOAD in a reproducible way, satisfying the basic principle of a REST interface. + +## Workflows +The typical operational workflows can be described in the following way. +An expert having a specific payload to upload over time (a sort of conditions type) should proceed with the following +steps in order to upload to CREST. + +- Create a `TAG` if it does not exists. +- Upload `IOV` and `PAYLOAD` to an existing `TAG`. +- Find an existing `TAG`. +- Load the `IOV` for a given `TAG`. +- Create a `GLOBALTAG`. +- Associate a given `TAG` to one or more `GLOBALTAG`s. +- Find a `GLOBALTAG` and retrieve the associated `TAG`s via the `GLOBALTAGMAP`. diff --git a/Dockerfile b/Dockerfile index 2fb1cc6a..4156422a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,48 +1,41 @@ # CrestDB -# -# VERSION CrestDB-1.0 - -# use the centos base image provided by dotCloud -# FROM openjdk:8u121-jdk -# FROM anapsix/alpine-java -# FROM openjdk:8u212-jre-alpine3.9 -#FROM openjdk:15-jdk-alpine -FROM adoptopenjdk/openjdk11:alpine-jre -MAINTAINER Andrea Formica - -ENV USR crest -ENV CREST_GID 208 - -ENV crest_version 1.0-SNAPSHOT -ENV crest_dir /home/${USR}/crest -ENV data_dir /home/${USR}/data -#ENV data_dir /data -ENV gradle_version 6.7 -ENV TZ GMT - -RUN addgroup -g $CREST_GID $USR \ - && adduser -S -u $CREST_GID -G $USR -h /home/$USR $USR - -RUN mkdir -p ${crest_dir} \ - && mkdir -p ${data_dir}/web \ - && mkdir -p ${data_dir}/dump \ - && mkdir -p ${data_dir}/logs - -## This works if using an externally generated war, in the local directory -ADD crestdb-web/build/libs/crest.war ${crest_dir}/crest.war -ADD web ${data_dir}/web -ADD logback.xml.crest ${data_dir}/logback.xml +FROM registry.cern.ch/docker.io/eclipse-temurin:23-alpine +LABEL maintainer="Andrea Formica" + +ARG CREST_USER_ID=1001 +ARG CREST_GROUP_ID=1001 + +# ===== Environment Variables ===== +ENV USR=crestsvc \ + TZ=GMT + +# Predefine paths (easier to modify) +ENV HOME=/home/${USR} \ + crest_dir=${HOME}/crest \ + data_dir=${HOME}/data \ + config_dir=${HOME}/config + +# ===== User & Group Setup ===== +# Create group and user in a single layer +RUN addgroup -g $CREST_GROUP_ID crest && \ + adduser -u $CREST_USER_ID -G crest -h $HOME -D $USR && \ + # Create all required directories in one RUN + mkdir -p ${crest_dir} ${config_dir} \ + ${data_dir}/web ${data_dir}/dump ${data_dir}/logs && \ + chown -R $USR:$CREST_GROUP_ID $HOME + +# ===== Application Setup ===== +# Add jar and entrypoint in separate layers (better caching) +ADD build/libs/crest.jar ${crest_dir}/crest.jar +COPY entrypoint.sh $HOME/ + +# ===== Permissions & Runtime Config ===== +RUN chmod +x $HOME/entrypoint.sh && \ + chown $USR:$CREST_GROUP_ID $HOME/entrypoint.sh ### we export only 1 directories.... VOLUME "${data_dir}" EXPOSE 8080 - -# copy the entrypoint -COPY ./entrypoint.sh /home/${USR} -COPY ./create-properties.sh /home/${USR} - -RUN chown -R $USR:$CREST_GID /home/${USR} - ### we set the user and the workdir.... USER ${USR} WORKDIR /home/${USR} diff --git a/Makefile b/Makefile index 817dcf2f..69a7c8b6 100644 --- a/Makefile +++ b/Makefile @@ -7,30 +7,30 @@ else SPECFILE = $(shell find . -maxdepth 1 -type f -name *.spec) endif -SPECFILE_NAME = $(shell awk '$$1 == "Name:" { print $$2 }' $(SPECFILE) ) -SPECFILE_VERSION = $(shell awk '$$1 == "Version:" { print $$2 }' $(SPECFILE) ) -SPECFILE_RELEASE = $(shell awk '$$1 == "Release:" { print $$2 }' $(SPECFILE) ) -TARFILE = $(SPECFILE_NAME)-$(SPECFILE_VERSION).tgz +#SPECFILE_NAME = $(shell awk '$$1 == "Name:" { print $$2 }' $(SPECFILE) ) +#SPECFILE_VERSION = $(shell awk '$$1 == "Version:" { print $$2 }' $(SPECFILE) ) +#SPECFILE_RELEASE = $(shell awk '$$1 == "Release:" { print $$2 }' $(SPECFILE) ) +#TARFILE = $(SPECFILE_NAME)-$(SPECFILE_VERSION).tgz DIST = $(shell rpm --eval %{dist}) -CREST_VERSION = 2.0 -CREST_RELEASE = $(shell sed -nr '/release=/ s/.*release=([^"]+).*/\1/p' $(PWD)/crestdb-web/src/main/resources/messages.properties) +CREST_VERSION = $(shell sed -nr '/version=/ s/.*version=([^"]+).*/\1/p' $(PWD)/src/main/resources/rpm.properties) +CREST_RELEASE = $(shell sed -nr '/release=/ s/.*release=([^"]+).*/\1/p' $(PWD)/src/main/resources/rpm.properties) +$(info CREST version : $(CREST_VERSION)) +$(info CREST release : $(CREST_RELEASE)) + TARGET_DIR = "crest-dist" -CREST_TARFILE = $(SPECFILE_NAME)-$(CREST_VERSION).tgz -CREST_WAR = $(shell find ./crestdb-web/ -maxdepth 3 -type f -name "crest.war") -CREST_IMAGE = "gitlab-registry.cern.ch/formica/crest:$(CREST_VERSION)" -##CREST_IMAGE = "crest-test" +CREST_JAR = $(shell find ./ -maxdepth 3 -type f -name "crest.jar") ## Commands MD = mkdir CP = cp DOCK = docker GRADLE = ./gradlew -sources: - ifeq ($(UNAME_S), Darwin) - $(info create tar for MACOSX) - tar -zcvf --exclude='.git' --exclude='.gitignore' --transform 's,^,$(SPECFILE_NAME)-$(SPECFILE_VERSION)/,' $(TARFILE) src/* - else - tar -zcvf $(TARFILE) --exclude-vcs --transform 's,^,$(SPECFILE_NAME)-$(SPECFILE_VERSION)/,' src/* - endif +#sources: +# ifeq ($(UNAME_S), Darwin) +# $(info create tar for MACOSX) +# tar -zcvf --exclude='.git' --exclude='.gitignore' --transform 's,^,$(SPECFILE_NAME)-$(SPECFILE_VERSION)/,' $(TARFILE) src/* +# else +# tar -zcvf $(TARFILE) --exclude-vcs --transform 's,^,$(SPECFILE_NAME)-$(SPECFILE_VERSION)/,' src/* +# endif clean: rm -rf build/ $(TARFILE) @@ -40,7 +40,7 @@ rpm: sources srpm: sources rpmbuild -bs --define 'dist $(DIST)' --define "_topdir $(PWD)/build" --define '_sourcedir $(PWD)' $(SPECFILE) mrpm: - rpmbuild -bb --define "_topdir $(PWD)/build" --define "_sourcedir $(PWD)" --define "version $(COOLR_VERSION)" --define "release $(COOLR_RELEASE)" $(SPECFILE) + rpmbuild -bb --define "_topdir $(PWD)/build" --define "_sourcedir $(PWD)" --define "version $(CREST_VERSION)" --define "release $(CREST_RELEASE)" $(SPECFILE) dist: rm -rf $(TARGET_DIR) @@ -49,7 +49,7 @@ dist: build: clean $(GRADLE) clean build package: dist - $(CP) $(CREST_WAR) $(TARGET_DIR)/crest.war + $(CP) $(CREST_JAR) $(TARGET_DIR)/crest.jar $(CP) ./logback.xml.crest $(TARGET_DIR)/logback.xml $(CP) ./javaopts.properties.rpm $(TARGET_DIR)/javaopts.properties $(CP) ./entrypoint.sh $(TARGET_DIR)/entrypoint.sh diff --git a/README.md b/README.md index 3208af20..0ee07065 100644 --- a/README.md +++ b/README.md @@ -1,294 +1,24 @@ -![Java CI with Gradle](https://github.com/HSF/Crest/workflows/Java%20CI%20with%20Gradle/badge.svg?event=push) +# JAX-RS/Jersey server with OpenAPI -#### Author: A.Formica, R.Sipos -##### Date of last development period: 2019/01/13 -##### Recent additions: new api methods for uploads of iov+payload, a web-ui in vuejs. -``` - Copyright (C) 2016 A.Formica, R.Sipos - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -``` -# Table of Contents -1. [Description](#description) -2. [Installation](#installation) -3. [Build instructions](#build-instructions) -4. [Run the server](#run-the-server) -5. [Swagger](#swagger) -6. [Docker](#docker) -7. [Openshift](#openshift) -8. [Clients](#clients) - - -## Description -Test project for the implementation of a generic purpose conditions database for physics experiment. -This server was generated by the [swagger-codegen](https://github.com/swagger-api/swagger-codegen) project. By using the -[OpenAPI-Spec](https://github.com/swagger-api/swagger-core/wiki) from a remote server, you can easily generate a server stub. This -project is an example of building a swagger-enabled JAX-RS server. Some tests were also done to provide a Resteasy implementation. - -The prototype uses [Spring framework](https://spring.io) and the REST services are implemented via [Jersey](https://jersey.java.net). - -The prototype runs as a microservice using `spring-boot`. By default it uses an embedded [undertow](http://undertow.io) servlet container, but others like [tomcat](https://tomcat.apache.org) or [jetty](https://www.eclipse.org/jetty/) can be easily used instead of [undertow](http://undertow.io). - - -You can directly download an `html` file from gitlab to visualize the API documentation on your browser: -``` -https://drf-gitlab.cea.fr/api/v4/projects/523/repository/files/doc%2Findex.html/raw?ref=master -``` - - -## Installation -Download the project from gitlab (example below is using `https`): -``` -git clone https://gitlab.cern.ch/formica/crest.git -``` -or -``` -git clone https://github.com/HSF/Crest.git -``` -or -``` -git clone https://drf-gitlab.cea.fr/svom/common/crest.git -git checkout cms-v0 -``` -if you are taking the github version. -This will create a directory `crest` in the location where you run the git command. - -## Build instructions -You need to have java >= 8 installed on your machine. If you have also [gradle](https://gradle.org) (version 5) you can build the project using the following command from the root project directory (`crest`): -``` -gradle clean build -``` -This command will generate a war (java web archive) file in : `crestdb-web/build/libs/crest.war`. -In case gradle is not installed on your machine, you can run the wrapper delivered with the project: -``` -./gradlew clean build -``` -If you want to select a specific JVM when you run gradle you can use a command like this: -``` -gradle clean build -Dorg.gradle.java.home=/path to jvm/11.0/ -``` -Be careful to checkout the correct branch. - -| ATLAS | CMS | SVOM | -| ------ | ------ | ------ | -| master | cms-v0 | cms-v0 | -| contains tag meta information | standard API | standard API | - -## Run the server -This section is under maintenance. - -The server will use by default an embedded `undertow` web server. - -The server need by definition to have a database connection in order to store the conditions data. The database connections are defined in the files `./crestdb-web/src/main/resources/application-.yml`. This file present different set of properties which are chosen by selecting a specific spring profile when running the server. The file should be edited if you are administering the conditions database in order to provide an appropriate set of parameters. - -If you do not have any remote database available you should use the default spring profile. - -The set of default properties to run the server is defined in `config/application.properties` which will be read by spring when starting the server. The file there will use the `default` spring profile and a local database instance `h2database` where to store the data (it is a sort of `sqlite` file). -It is essential to set appropriate parameters in this file. An example -is provided to start up a default profile with local h2 database. - -#### Oracle -``` -spring.profiles.active=oracle -user.timezone=GMT -crest.api.name=/crestapi -crest.web.static=/tmp/data/web -crest.dump.dir=/tmp/data/dump -crest.log.dir=/tmp/data/dump -crest.db.password=somepassword -crest.db.user=ATLAS_PHYS_COND_01_W -crest.port=8090 -crest.db.url=jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=cman1-atlas.cern.ch)(PORT=10500))(LOAD_BALANCE=on)(ENABLE=BROKEN)(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=int8r.cern.ch))) -crest.db.schema=ATLAS_PHYS_COND_01 -``` -#### Postgres -``` -spring.profiles.active=postgres -user.timezone=GMT -crest.web.static=/tmp/data/web -crest.dump.dir=/tmp/data/dump -crest.log.dir=/tmp/data/logs -management.endpoint.health.show-details=ALWAYS -crest.db.url=jdbc:postgresql://postgres-host:5432/crestdb -crest.db.user=someuser -crest.db.password=somepassword -``` - -### Start -To start the server you can simply run: - -``` -./entrypoint.sh -``` -This script is the same that is used by the docker container (when packaging the server via the `Dockerfile`). - -We provide the following commands as examples for alternative way (not maintained anymore): -``` -cd crestdb-web -$ gradle bootRun "-Dspring.profiles.active=oracle" "-Dcrest.db.password=xxx" -``` -or -``` -$java -Dspring.profiles.active=oracle -Dcrest.db.password=xxx -jar crestdb-web/build/libs/crest.war -``` - -### Activate security -This part is obsolete. We are working on OAuth2 implementation. - -To activate security you need to build the war file including the key-store. The file should go into /src/main/resources together with a complete ldap.properties file in which you need to set the manager password. -These are not detailed instructions, it is more a reminder. - -``` -java -Dstore.password=xxx -Dkey.password=yyy -Dcrest.db.password=ddd -Dcrest.dump.dir=/data/data/dump -Dcrest.web.static=/data/data/web -Dspring.profiles.active=prod -jar crestdb-web/build/libs/crest.war -``` -The prod profile is using CERN ldap. Here is an example of ldap properties. - -``` -USER_SEARCH_BASE="DC=cern,DC=ch" -USER_DN_PATTERNS="CN={0},OU=Users,DC=cern,DC=ch" -GROUP_SEARCH_BASE="OU=e-groups,OU=Workgroups,DC=cern,DC=ch" -GROUP_SEARCH_FILTER="member={0}" -GROUP_ROLE_ATTRIBUTE=cn -MANAGER_DN="CN=formica,OU=Users,OU=Organic Units,DC=cern,DC=ch" -MANAGER_PASSWORD=xxx -LDAP_AUTHENTICATOR_URL=ldaps://cerndc.cern.ch:636 -ACCESS=hasRole('atlas-database') -``` -In order to test security you can try to use curl: - -``` -curl -k -u user:password -X GET https://localhost:8443/crestapi/globaltags -``` -The -k should skip verification on the certificate. - -In order to connect to the ldap server we need to have the truststore correctly set and with an alias corresponding to the `cerndc.cern.ch` certificate. Some java properties need to be set for this: - -``` --Djavax.net.ssl.trustStore=/ssl-crest-server.jks -Djavax.net.ssl.trustStorePassword=xxx -Djavax.net.debug=ssl -``` -Be careful that the properties defined in the `application.yml` do not work for the truststore. -In order to add certificates to the truststore you can proceed in the following way: - -``` -echo -n | openssl s_client -connect cerndc.cern.ch:636 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' > /tmp/examplecert.crt -openssl x509 -in /tmp/examplecert.crt -text -``` -This will retrieve the server side certificate of the host you want to connect to for authentication. +## Overview +This server was generated by the [OpenAPI Generator](https://openapi-generator.tech) project. By using an +[OpenAPI-Spec](https://openapis.org), you can easily generate a server stub. -``` -keytool -import -trustcacerts -keystore ./crestdb-web/src/main/resources/ssl-crest-server.jks -storepass xxxxx -noprompt -alias cern -file /tmp/examplecert.crt -``` -This instead will add the certificate to the truststore (which in our case is the same file). -The truststore has been created using a command like: +This is an example of building a OpenAPI-enabled JAX-RS server. +This example uses the [JAX-RS](https://jax-rs-spec.java.net/) framework. +Jersey is used as JAX-RS implementation, `io.swagger:swagger-jersey2-jaxrs` is used to derive the OpenAPI Specification from the annotated code. -``` -keytool -genkey -alias crest_localhost_sslserver -keyalg RSA -keysize 2048 -validity 700 -keypass xxx -storepass xxxx -keystore ssl-crest-server.jks -``` - -## Swagger -You can view the swagger listing here (hopefully the server will be up!): +To run the server, please execute the following: ``` -http://crest-undertow.web.cern.ch/crestapi/swagger.json +mvn clean package jetty:run ``` -and if you want to play with the server using the swagger-ui you can access it here: -``` -http://crest-undertow.web.cern.ch/ext/web/ui/index.html -``` - -Note that in principle you can get the same links working (a part from the hostname) if you run the server locally. - -The same kind of visualisation is available directly in gitlab when accessing the [specification file](./swagger_schemas/swagger/json/crestdb_full.json). - -### Swagger code generation -In order to regenerate the API we use the JSON schemas and templates which are store in the directories: +You can then view the OpenAPI v2 specification here: ``` -./swagger_schemas -./templates +http://localhost:8080/api/swagger.json ``` -To run code generation some scripts can be used as examples (`./scripts`). -The server stub generation is implemented as well as a gradle task: - -``` -./gradlew generateSwaggerCode -``` - -## Docker -You can build a container using - -``` -docker build -t crest:1.0 . -``` -You can run the container using - -``` -docker run --env-file .environment -p 8080:8080 -d crest:1.0 -``` -or - -``` -docker run --env-file .environment -p 8080:8080 -v /mnt/data/dump:/data/dump -v /mnt/data/web:/data/web --net=host -d crest:test -``` -In the last example we have been mounting external volumes. These are useful for the swagger-ui and the possibility to dump a tag in a file system based structure. You can use the swagger-ui version that is provided within this project in the directory - -``` -./web/ui/ -``` -A special note about the file `.environment` . You need to have this file to set variables which are used at the startup of the server. Some of the variables are already provided in the version in git, but other are not. For example, to access Oracle at CERN (for the moment only integration cluster contains a crest schema) you need to have the variable `crest.db.password=xxxxx` correctly set for a writer account. -If you use `spring.profiles.active=default` you will have an h2 database created in `jdbc:h2:/tmp/cresth2;DB_CLOSE_ON_EXIT=FALSE`. - -You can connect to a running container using commands like: - -``` -docker exec -i -t infallible_stonebraker /bin/bash -``` - -In order to push an image created into the CERN gitlab registry you need to login: -``` -docker login -u formica gitlab-registry.cern.ch -``` -You can choose another registry. -The push command work only if the login is successful. -### Swarm -As an example for a deployment in a swarm look at `./swarm/docker-compose.yml`. -In the same repository there is a script to help in generating the config maps needed and the secret. -An `application.properties` file should be created in order to run the server. Examples are shown above. - -## Openshift -We gather here some notes on openshift deployment via gitlab-ci. These notes are for usage inside CERN. -### Constraints -For the moment in order for the deployment to work we need to have a public access to the gitlab project. -### Problems -After committing a tag it seems that the deploy to openshift fails. -TO BE DONE. - -## Clients -We have been merging our clients in this repository with the contribution of some colleagues from *Juno* collaboration. We have now work in progress in the following areas (available code can be seen in *crestdb-client* repository). -> This is work in progress...documentation needs to be improved.... - -### Python -A small client is available in `crestdb-client/python/cli` that can be installed via `pip`. -One can also generate a python client via swagger. -### C++ -Ask Juno colleagues. -### gatling -Generated via swagger. Used for testing REST API. -### qt5cpp -Generated via swagger. This is just a demo. - -In addition we have recently added a Web GUI in *VueJS*. The project can be found in *web-ui/crest-ui*. -In order to run it one can simply follow the readme file. It can use for development purpose *npm* and *node*. +Note that if you have configured the `host` to be something other than localhost, the calls through +swagger-ui will be directed to that host and not localhost! \ No newline at end of file diff --git a/USAGE.md b/USAGE.md new file mode 100644 index 00000000..adae5649 --- /dev/null +++ b/USAGE.md @@ -0,0 +1,90 @@ +#### Author: A.Formica +# Table of Contents +1. [Introduction](#introduction) +2. [Deployment](#deployment) +2. [Tagging](#tagging) +3. [IoV](#iov) +4. [Payload](#payload) +5. [Global Tag](#global-tag) +6. [Global Tag Map](#global-tag-map) +7. [Examples](#examples) + +## Introduction +The main concepts behind *CREST* are the following: +- Tags: defined by a name, a payload type and time type. The time type indicates what is used in IoV. +- IoV: defined by a tag, a start time and payload hash (sha256 in general). The start time indicates the beginning of the validity of the payload. +- Payload: defined by a hash (sha256 in general) and a content. The content is a binary blob. +- Global Tag: a container of Tags, defined by a name. +- Global Tag Map: a collection of Tags to Global Tag mappings. The mapping is defined by a tag name, a record and a label. + +## Deployment +The list of available servers is the following: + +| Server | URL | APID | Description | +| ------ |------------------------------------|----------|----------------------------------------------------| +| Development | http://crest-j23.cern.ch:8080 | api-v5.0 | Development server, use ATLAS_PHYS_COND_01 @ INT8R | +| Production | https://crest.cern.ch | api-v5.0 | Production read only server, use ATLAS_PHYS_COND @ INT8R | +| Production | http://crest-03.cern.ch:9090 | api-v5.0 | Production read/write server, use ATLAS_PHYS_COND @ INT8R | +| Production | http://crest-04.cern.ch | api-v5.0 | Production read/write server, use ATLAS_PHYS_COND @ INT8R | +| Test | http://atlaf-alma9-02.cern.ch:8080 | api-v6.0 | Test server, use ATLAS_PHYS_COND_01 @ INT8R | + +There are haproxy in P1 to access the production servers. The haproxy is configured to use the following servers: +- _atlashap02-atcn:8081_ : access the read-write server crest-03.cern.ch +- _atlashap01-atcn:8080_ : access the read-only server crest.cern.ch + +## Tagging +Here are the main operations related to tagging: + + * create a tag: `POST /tags` + * get a tag: `GET /tags/{name}` + * get all tags: `GET /tags` (paginated), use several parameters to filter the results. + +## IoV +Here are the main operations related to IoV: + + * create an iov: `POST /iovs`, a list of iovs can be created at once. + * get all iovs: `GET /iovs` (paginated), use several parameters to filter the results. + +## Payload +Here are the main operations related to payloads: + + * create a payload: `POST /payloads` + * get a payload: `GET /payloads/{hash}` + +## Global Tag +Here are the main operations related to global tags: + + * create a global tag: `POST /globaltags` + * get a global tag: `GET /globaltags/{name}` + * get all global tags: `GET /globaltags` (paginated), use several parameters to filter the results. + +## Global Tag Map +Here are the main operations related to global tag maps: + + * create a global tag map: `POST /globaltagmaps` + +## Examples +A user needs to create a tag for a payload of type `mytype` and time type `run-lumi`. +The user creates the tag with the following command: +``` +curl -X POST "http://localhost:8080/tags" -H "accept: application/json" -H "Content-Type: application/json" -d "{ \"name\": \"mytag\", \"payloadType\": \"mytype\", \"timeType\": \"run-lumi\"}" +``` +The user creates a list of payloads inside a tag, providing the iov associated with the following command: +``` +curl -X POST "http://localhost:8080/payloads" -H "accept: application/json" -H "Content-Type: application/json" -d "" +``` +The user creates a global tag with the following command: +``` +curl -X POST "http://localhost:8080/globaltags" -H "accept: application/json" -H "Content-Type: application/json" -d "{ \"name\": \"myglobaltag\", \"tags\": [ { \"name\": \"mytag\", \"record\": \"myrecord\", \"label\": \"mylabel\" } ]}" +``` +The user creates a global tag map with the following command: +``` +curl -X POST "http://localhost:8080/globaltagmaps" -H "accept: application/json" -H "Content-Type: application/json" -d "{ \"globalTagName\": \"myglobaltag\", \"tagName\": \"mytag\", \"record\": \"myrecord\", \"label\": \"mylabel\"}" +``` +The user gets the global tag with the following command: +``` +curl -X GET "http://localhost:8080/globaltags/myglobaltag" -H "accept: application/json" +``` +A user can copy a payload from another tag to a new tag with the following command: +- get the iov corresponding to the payload from the old tag. +- insert the iov in the new tag, after changing eventually the start time (since). diff --git a/build.gradle b/build.gradle index ce8d4f48..666f9495 100644 --- a/build.gradle +++ b/build.gradle @@ -1,187 +1,250 @@ -buildscript { - repositories { - jcenter() +plugins { + id 'java' + id 'org.springframework.boot' version '3.4.3' + id 'io.spring.dependency-management' version '1.1.6' + id("org.openapi.generator") version "7.8.0" + id 'jacoco' + id 'checkstyle' + id "org.sonarqube" version "6.0.1.5171" +} + +group = 'hep.crest' +version = '6.1.0' + +java { + toolchain { + languageVersion = JavaLanguageVersion.of(23) } +} + +springBoot { + // This statement tells the Gradle Spring Boot plugin + // to generate a file + // build/resources/main/META-INF/build-info.properties + // that is picked up by Spring Boot to display + // via /info endpoint. + buildInfo() +} - dependencies { - classpath 'io.github.swagger2markup:swagger2markup-gradle-plugin:1.3.3' +bootJar { + archiveFileName = 'crest.jar' +// metaInf { from 'src/main/resources/spring.factories' } +} + +sourceCompatibility = '17' +targetCompatibility = '17' + +configurations { + compileOnly { + extendsFrom annotationProcessor } + querydsl.extendsFrom annotationProcessor } -plugins { - id "org.sonarqube" version "2.6" - id 'org.hidetake.swagger.generator' version '2.9.0' - id 'jacoco' +configurations.all { + exclude group: 'org.springframework.boot', module: 'spring-boot-starter-tomcat' + exclude group: 'org.slf4j', module: 'slf4j-simple' } -apply plugin: 'java' -apply plugin: 'maven' -apply plugin: 'io.github.swagger2markup' +checkstyleMain { + setExcludes(new HashSet(['**/hep/crest/data/**/Q*java', + '**/src/gen/**', + '**/hep/crest/data/repositories/externals/*', + '**/hep/crest/server/swagger/**', + '**/plugins/nats/**', + '**/plugin/nats/model/*java'])) +} -ext { - swagger_annotations_version = "1.5.13" - jackson_version = "2.7.5" - spring_version = "4.1.4.RELEASE" - springboot_version = "2.1.0.RELEASE" - junit_version = "4.12" - oltu_version = "1.0.1" - orika_version = "1.5.1" - querydsl_version = "4.2.1" +tasks.withType(Checkstyle) { + checkstyleTest.enabled = false + reports { + xml.required = false + html.required = true + } +} + +jacocoTestReport { + + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it, exclude: [ + "src/gen/**", + "hep/crest/server/data/**/Q*.*", + "hep/crest/server/swagger/api/*", + "hep/crest/server/swagger/model/*" + ]) + })) + } + reports { + xml.required = true + csv.required = false + html.outputLocation = layout.buildDirectory.dir('jacocoHtml') + } +} + + +sonar { + properties { + property "sonar.projectKey", "andreaformica_Crest" + property "sonar.organization", "andreaformica" + property "sonar.host.url", "https://sonarcloud.io" + } } repositories { - mavenCentral() + mavenCentral() +} + +ext { + swaggerDirectory = "$rootDir/openapi".toString() + swaggerSpecDirectory = "$swaggerDirectory".toString() + swaggerBundleDirectory = "$swaggerDirectory/bundle".toString() + swaggerJaxrsTemplateDirectory = "$swaggerDirectory/templates/JavaJaxRs".toString() +} + +task openApiBundle { + // Bundle the openapi spec files together, to mitigate a bug with the main openapi generator + // Bug: https://github.com/OpenAPITools/openapi-generator/issues/1976 + // For this to work, the swagger-cli (from the javascript world) needs to be installed + // with e.g: npm install -g @apidevtools/swagger-cli + inputs.files(fileTree(dir: swaggerSpecDirectory, include: '**/*.yml')) + outputs.file("$swaggerBundleDirectory/crestAtlasApi_all.yml".toString()) + + doLast { + exec { + workingDir swaggerSpecDirectory + commandLine 'redocly', 'bundle', '-o', "$swaggerBundleDirectory/crestApi_all.yml".toString(), "$swaggerSpecDirectory/crestAtlasApi_all.yml".toString() + } + } +} + +openApiGenerate { + generatorName = "jaxrs-jersey" + inputSpec = "$swaggerBundleDirectory/crestApi_all.yml".toString() + outputDir = "$projectDir".toString() + templateDir = swaggerJaxrsTemplateDirectory + apiPackage = "hep.crest.server.swagger.api" + modelPackage = "hep.crest.server.swagger.model" + configOptions = [ + dateLibrary: "java8", + hideGenerationTimestamp: "true", + useJakartaEe: "true" + ] +} + +task buildCrestPythonClient(type: org.openapitools.generator.gradle.plugin.tasks.GenerateTask){ + generatorName = "python" + inputSpec = "$swaggerBundleDirectory/crestApi_all.yml".toString() + outputDir = "$rootDir/pycrest".toString() + packageName = "hep.crest.client" + configOptions = [ + packageVersion: "6.1.0", + projectName: "hep-crest-client"] + +} + +task buildCrestMarkdownDoc(type: org.openapitools.generator.gradle.plugin.tasks.GenerateTask){ + generatorName = "markdown" + inputSpec = "$swaggerBundleDirectory/crestApi_all.yml".toString() + outputDir = "$rootDir/mddocs".toString() + packageName = "hep.crest.client" + configOptions = [ + packageVersion: "5.0.0", + projectName: "hep-crest-client"] + } dependencies { - swaggerCodegen 'io.swagger:swagger-codegen-cli:2.3.1' -} - -// Task to generate sources in build output directory -convertSwagger2markup { - swaggerInput file("swagger_schemas/swagger/json/crestdb_full.json").getAbsolutePath() - outputDir file("build/asciidoc") - config = ['swagger2markup.markupLanguage' : 'MARKDOWN', - 'swagger2markup.pathsGroupedBy' : 'TAGS', - 'swagger2markup.interDocumentCrossReferencesEnabled' : true] -} - -swaggerSources { - crestdb { - inputFile = file('swagger_schemas/swagger/json/crestdb_full.json') - code { - language = 'jaxrs' - templateDir = file('./templates/java/JavaJaxRs') - configFile = file('swagger_schemas/swagger/json/crestdb-config.json') - } - } - crestdb_client { - inputFile = file('swagger_schemas/swagger/json/crestdb.json') - code { - language = 'python' - //templateDir = file('./templates/2.3.0/python-client/model') - configFile = file('swagger_schemas/swagger/json/crestdb-pyclient-config.json') - } - } -} - -configurations.compile.each { - println "compile: $it" -} - -allprojects { - apply plugin: 'maven' - apply plugin: 'jacoco' - apply plugin: 'java' - - group = 'hep.crest' - version = '2.0-SNAPSHOT' - repositories { - mavenCentral() - } - jacoco { - //toolVersion = '0.8.4' - reportsDir = file("$buildDir/reports/jacoco") - } - jacocoTestReport { - reports { - xml.enabled true - csv.enabled false - html.destination file("${buildDir}/jacocoHtml") - } - } -} - -// An example for sonarqube configuration: to run the code on a local sonarqube instance you should -// use commands like : ./gradlew sonarqube -Dsonar.host.url=http://localhost:9000 -Dsonar.login=180fe2ea3c768359eb5fa680f7bbe6667ceeb238 -sonarqube { - properties { - property "sonar.host.url", "http://localhost:9000" - property "sonar.login", "d7bef5b1a70c8f5812a14c196d9920c65ecbc28c" - property "sonar.projectKey", "crest" - property "sonar.projectName", "crest" - property "sonar.junit.reportPaths", "build/test-results/test" - property 'sonar.coverage.exclusions', "**/hep/crest/swagger/model/*" - property "sonar.exclusions", "**/hep/crest/swagger/model/*,**/hep/crest/server/swagger/api/*,**/hep/crest/data/dialect/*" - } -} - -subprojects { - sourceCompatibility = 11 - targetCompatibility = 11 - ext { - swagger_annotations_version = "1.5.13" - jackson_version = "2.10.5" - springboot_version = "2.3.1.RELEASE" - junit_version = "4.12" - oltu_version = "1.0.1" - orika_version = "1.5.1" - jersey2_version = "2.22.2" - swagger_core_version = "1.5.13" - querydsl_version = "4.4.0" - server = "undertow" - } - - tasks.withType(JavaCompile) { - options.encoding = "UTF-8" - } - - repositories { - mavenLocal() - mavenCentral() - maven { url "https://repo.spring.io/milestone" } - maven { url "https://jcenter.bintray.com/" } - maven { url "https://hibernate-sqlite.googlecode.com/svn/trunk/mavenrepo" } - maven { url "https://repo.maven.apache.org/maven2" } - } - - configurations { - compile.exclude module: "tomcat-jdbc" - } - - dependencies { - // JAX-B dependencies for JDK 9+ - implementation "jakarta.xml.bind:jakarta.xml.bind-api:2.3.2" - implementation "org.glassfish.jaxb:jaxb-runtime:2.3.2" - // javassist for JDK 9+ - implementation "org.javassist:javassist:3.23.1-GA" - // swagger - implementation "io.swagger:swagger-annotations:$swagger_annotations_version" - implementation "io.swagger:swagger-core:$swagger_core_version" - implementation "io.swagger:swagger-jersey2-jaxrs:$swagger_core_version" - implementation group: 'ma.glasnost.orika', name: 'orika-core', version: orika_version - - - implementation "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:$jackson_version" - implementation "com.fasterxml.jackson.core:jackson-core:$jackson_version" - implementation "com.fasterxml.jackson.core:jackson-annotations:$jackson_version" - implementation "com.fasterxml.jackson.core:jackson-databind:$jackson_version" - implementation "com.fasterxml.jackson.datatype:jackson-datatype-joda:$jackson_version" - - // DATABASES - //implementation group: 'com.oracle', name: 'ojdbc7', version:'12.1.0.2' - implementation files("${project.rootDir}/jarlib/ojdbc8-12.2.0.1.jar") -// https://mvnrepository.com/artifact/org.xerial/sqlite-jdbc - implementation group: 'org.xerial', name: 'sqlite-jdbc', version: '3.25.2' - implementation group: 'org.postgresql', name: 'postgresql', version: '42.2.8' - implementation group: 'mysql', name: 'mysql-connector-java', version: '5.1.13' - implementation("com.h2database:h2") - implementation("commons-io:commons-io:2.6") - implementation group: 'commons-codec', name: 'commons-codec', version: '1.11' - - } - - task wrapper(type: Wrapper) { - gradleVersion = '6.4' //version required - } - - // Clean files and directories used for unit testing - task purgedb(type: Delete) { - delete fileTree('/tmp') { - include 'crest*.db' - include 'crest*sqlite*' - include 'tagtar*' - } - delete '/tmp/cdms' - } - test.dependsOn purgedb + compileOnly 'org.projectlombok:lombok:1.18.34' + annotationProcessor 'org.projectlombok:lombok:1.18.34' + // Mappper + implementation 'org.mapstruct:mapstruct:1.6.2' + annotationProcessor 'org.mapstruct:mapstruct-processor:1.6.2' + // Lombok-MapStruct Binding + implementation 'org.projectlombok:lombok-mapstruct-binding:0.2.0' + annotationProcessor 'org.projectlombok:lombok-mapstruct-binding:0.2.0' + // QueryDSL + implementation 'com.querydsl:querydsl-jpa:5.0.0:jakarta' // QueryDSL for JPA + annotationProcessor 'com.querydsl:querydsl-apt:5.0.0:jakarta' + annotationProcessor "jakarta.persistence:jakarta.persistence-api" + annotationProcessor "jakarta.annotation:jakarta.annotation-api" + // Spring Boot + implementation 'org.springframework.boot:spring-boot-starter-actuator' + implementation 'org.springframework.boot:spring-boot-starter-data-jpa' + implementation 'org.springframework.boot:spring-boot-starter-data-redis' + // implementation 'org.springframework.boot:spring-boot-starter-data-rest' +// Exclude default Tomcat + implementation('org.springframework.boot:spring-boot-starter-web') { + exclude group: 'org.springframework.boot', module: 'spring-boot-starter-tomcat' + } + // Add Undertow as the embedded server + implementation 'org.springframework.boot:spring-boot-starter-undertow' + + implementation ('org.springframework.boot:spring-boot-starter-jersey'){ + exclude group: 'org.springframework.boot', module: 'spring-boot-starter-tomcat' + } + implementation 'org.springframework.boot:spring-boot-starter-security' + implementation('org.springframework.boot:spring-boot-starter-oauth2-client') + implementation 'org.springframework.boot:spring-boot-starter-oauth2-resource-server' + + // Jackson libraries + implementation "com.fasterxml.jackson.jakarta.rs:jackson-jakarta-rs-json-provider:2.18.3" + implementation group: 'org.glassfish.jersey.media', name: 'jersey-media-multipart', version: '3.1.0' + // Keycloak + // implementation 'org.keycloak:keycloak-spring-boot-starter:22.0.1' + // Hibernate + // Hibernate Core + implementation 'org.hibernate:hibernate-core:6.5.3.Final' // Adjust to your Hibernate version + implementation 'org.hibernate:hibernate-jpamodelgen:6.5.3.Final' // For JPA model generation, if needed + + + // Swagger + implementation group: 'io.swagger.core.v3', name: 'swagger-annotations', version: '2.2.7' + implementation group: 'io.swagger.core.v3', name: 'swagger-models', version: '2.2.7' + implementation 'org.apache.commons:commons-text:1.10.0' + + // Caching + implementation 'com.github.ben-manes.caffeine:caffeine:3.1.8' + + implementation 'com.oracle.database.jdbc:ojdbc11' + runtimeOnly 'org.postgresql:postgresql' + runtimeOnly 'com.h2database:h2' // Add H2 database dependency + runtimeOnly 'net.logstash.logback:logstash-logback-encoder:7.4' + + testImplementation "org.mockito:mockito-inline:+" + testImplementation 'org.springframework.boot:spring-boot-starter-test' + testRuntimeOnly 'org.junit.platform:junit-platform-launcher' + testRuntimeOnly 'com.h2database:h2' // Add H2 database dependency + testImplementation 'org.projectlombok:lombok:1.18.34' + testImplementation "org.testcontainers:testcontainers:1.20.4" + testImplementation 'com.redis:testcontainers-redis:2.2.2' + testAnnotationProcessor 'org.projectlombok:lombok:1.18.34' +} + +tasks.named('test') { + useJUnitPlatform() +} + +sourceSets { + main { + java { + // srcDirs = ['src/main/java', 'src/gen/java'] + srcDirs = ['src/main/java', 'src/gen/java', 'plugins/timeseries/java', 'plugins/tzero/java'] + } + } + // println main.output.classesDir + test { + java { + srcDirs = ['src/test/java'] + } + } +} + +// Clean files and directories used for unit testing +task purgedb(type: Delete) { + delete fileTree('/tmp') { + include 'crest*.db' + include 'crest*sqlite*' + } } + +test.dependsOn purgedb diff --git a/charts/Chart.yaml b/charts/Chart.yaml new file mode 100644 index 00000000..2f14413e --- /dev/null +++ b/charts/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: atlas-crest +description: Crest Helm chart + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 1.0.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "6.1.1" diff --git a/charts/templates/NOTES.txt b/charts/templates/NOTES.txt new file mode 100644 index 00000000..fe7c0da0 --- /dev/null +++ b/charts/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "crest.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch its status by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "crest.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "crest.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "crest.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/charts/templates/_helpers.tpl b/charts/templates/_helpers.tpl new file mode 100644 index 00000000..2a0fd175 --- /dev/null +++ b/charts/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "crest.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "crest.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "crest.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "crest.labels" -}} +helm.sh/chart: {{ include "crest.chart" . }} +{{ include "crest.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "crest.selectorLabels" -}} +app.kubernetes.io/name: {{ include "crest.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "crest.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "crest.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/charts/templates/deployment.yaml b/charts/templates/deployment.yaml new file mode 100644 index 00000000..29cef009 --- /dev/null +++ b/charts/templates/deployment.yaml @@ -0,0 +1,76 @@ +# The crest deploymentconfig +# This configures the docker image to be run, volumes to be mounted, config, +# etc. The effect of this deploymentconfig is to start a POD running the +# crest server +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "crest.fullname" . }} + labels: + {{- include "crest.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "crest.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "crest.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "crest.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: crest-server + containerPort: {{ .Values.container.port }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.container.env }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/templates/ingress.yaml b/charts/templates/ingress.yaml new file mode 100644 index 00000000..91a2ee8d --- /dev/null +++ b/charts/templates/ingress.yaml @@ -0,0 +1,44 @@ +{{- if .Values.ingress.enabled -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "crest.fullname" . }} + labels: + {{- include "crest.labels" . | nindent 4 }} + namespace: default + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.ingress.className }} + ingressClassName: {{ . }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- with .pathType }} + pathType: {{ . }} + {{- end }} + backend: + service: + name: {{ include "crest.fullname" $ }} + port: + number: {{ $.Values.service.port }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/templates/pvc.yaml b/charts/templates/pvc.yaml new file mode 100644 index 00000000..34e4da35 --- /dev/null +++ b/charts/templates/pvc.yaml @@ -0,0 +1,11 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: scratch-vol +spec: + storageClassName: meyrin-cephfs + accessModes: + - ReadWriteMany + resources: + requests: + storage: 100Gi diff --git a/charts/templates/service.yaml b/charts/templates/service.yaml new file mode 100644 index 00000000..5f47010a --- /dev/null +++ b/charts/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "crest.fullname" . }} + labels: + {{- include "crest.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: crest-server + protocol: TCP + name: crest-http + selector: + {{- include "crest.selectorLabels" . | nindent 4 }} diff --git a/charts/templates/tests/test-connection.yaml b/charts/templates/tests/test-connection.yaml new file mode 100644 index 00000000..1b231911 --- /dev/null +++ b/charts/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "crest.fullname" . }}-test-connection" + labels: + {{- include "crest.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "crest.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/charts/values.yaml b/charts/values.yaml new file mode 100644 index 00000000..8ec27423 --- /dev/null +++ b/charts/values.yaml @@ -0,0 +1,156 @@ +# Default values for crest. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ +replicaCount: 1 + +# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ +image: + repository: registry.cern.ch/crest/crest21-service + # This sets the pull policy for images. + pullPolicy: Always + # Overrides the image tag whose default is the chart appVersion. + tag: "6.2" + +# This is for the secretes for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ +imagePullSecrets: [] +# This is to override the chart name. +nameOverride: "" +fullnameOverride: "" + +#This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/ +serviceAccount: + # Specifies whether a service account should be created + create: false + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +# This is for setting Kubernetes Annotations to a Pod. +# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +podAnnotations: {} +# This is for setting Kubernetes Labels to a Pod. +# For more information checkout: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ +podLabels: {} + +podSecurityContext: + fsGroup: 208 + +securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + +# This is for setting up a service more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/ +service: + # This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: ClusterIP + # This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports + port: 80 + +container: + port: 8080 + env: + - name: TZ + value: "Europe/Paris" + - name: TNS_ADMIN + value: /home/crestsvc + +# This block is for setting up the ingress for more information can be found here: https://kubernetes.io/docs/concepts/services-networking/ingress/ +ingress: + enabled: true + className: "nginx" + annotations: + kubernetes.io/ingress.class: nginx + kubernetes.io/tls-acme: "true" + nginx.ingress.kubernetes.io/upstream-hash-by: "$request_uri" + cert-manager.io/cluster-issuer: letsencrypt + hosts: + - host: atlas-crest-okd.web.cern.ch + paths: + - path: / + pathType: Prefix + +resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: + cpu: 4 + memory: 4Gi + requests: + cpu: 500m + memory: 2Gi + +# This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ +livenessProbe: + httpGet: + path: /actuator/health + port: crest-server + initialDelaySeconds: 30 + periodSeconds: 30 +readinessProbe: + httpGet: + path: / + port: crest-server + +#This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/ +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +# Additional volumes on the output Deployment definition. +volumes: +- name: crest-volume-data + persistentVolumeClaim: + claimName: scratch-vol + readOnly: false +- name: crest-config-volume + configMap: + name: crest-config +- name: crest-db-auth-volume + secret: + secretName: crest-phys-cond-secret + +# Additional volumeMounts on the output Deployment definition. +volumeMounts: +- name: crest-volume-data + mountPath: /home/crestsvc/data +- name: crest-config-volume + readOnly: true + mountPath: /home/crestsvc/config/application.properties + subPath: crest-application-properties +- name: crest-config-volume + readOnly: true + mountPath: /home/crestsvc/javaopts.properties + subPath: crest-java-opts +- name: crest-config-volume + readOnly: true + mountPath: /home/crestsvc/tnsnames.ora + subPath: crest-tnsnames +- name: crest-config-volume + readOnly: true + mountPath: /home/crestsvc/logback.xml + subPath: crest-logging-config +- name: crest-db-auth-volume + mountPath: /run/secrets/crest-phys-cond + subPath: crest-phys-cond + readOnly: true + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/config/application.properties.example b/config/application.properties.example index f6ce1745..82c354ed 100644 --- a/config/application.properties.example +++ b/config/application.properties.example @@ -1,8 +1,10 @@ spring.profiles.active=default user.timezone=GMT -crest.api.name=/api +crest.api.name=/api-v6.0 crest.web.static=/tmp/data/web crest.dump.dir=/tmp/data/dump crest.log.dir=/tmp/data/dump crest.server.security=none +crest.redis.host=localhost +crest.redis.port=6379 management.endpoint.health.show-details=ALWAYS diff --git a/crest.service b/crest.service index fdba60b1..9bfc4f23 100644 --- a/crest.service +++ b/crest.service @@ -1,10 +1,10 @@ [Unit] -Description=COOLR REST application (SpringBoot v2.3.0) +Description=CREST REST application (SpringBoot v3.3.4) After=syslog.target [Service] -User=coolr -ExecStart=/usr/local/share/coolr/entrypoint.sh +User=crest +ExecStart=/usr/local/share/crest/entrypoint.sh SuccessExitStatus=143 Restart=always RestartSec=20 diff --git a/crest.spec b/crest.spec index 36e4f428..f9557ae0 100644 --- a/crest.spec +++ b/crest.spec @@ -20,8 +20,8 @@ getent passwd crest >/dev/null || \ %install rm -rf $RPM_BUILD_ROOT -mkdir -p $RPM_BUILD_ROOT/usr/local/share/crest -cp -p %{_sourcedir}/coolR-web/build/libs/crest.war $RPM_BUILD_ROOT/usr/local/share/crest +mkdir -p $RPM_BUILD_ROOT/usr/local/share/crest/config +cp -p %{_sourcedir}/build/libs/crest.jar $RPM_BUILD_ROOT/usr/local/share/crest cp -p %{_sourcedir}/config/application.properties $RPM_BUILD_ROOT/usr/local/share/crest cp -p %{_sourcedir}/crest.service $RPM_BUILD_ROOT/usr/local/share/crest cp -p %{_sourcedir}/entrypoint.sh $RPM_BUILD_ROOT/usr/local/share/crest diff --git a/crestdb-client/curl/check-payload.sh b/crestdb-client/curl/check-payload.sh new file mode 100644 index 00000000..45170a90 --- /dev/null +++ b/crestdb-client/curl/check-payload.sh @@ -0,0 +1,26 @@ +# CREST VS COOL Payload Checker +# Requires the 2 CURL commands to return the same IOV from both system + +cool_curl="http://atlas-coolr-api.web.cern.ch/api/payloads?schema=ATLAS_COOLOFL_TILE&node=/TILE/OFL02/CALIB/CIS/LIN&tag=TileOfl02CalibCisLin-RUN2-HLT-UPD1-00&since=1888943796649984&until=1917896171192319" +crest_curl="http://crest-undertow-api.web.cern.ch:80/api-v4.0/payloads/d9c6411b58e89619e0ffd55324e1a4c3dacd29a285f51ca12c49f933332821b2?format=BLOB" + +# Get files from both systems +curl -s -o cool.json $cool_curl +curl -s -o crest.json $crest_curl + +# Check if the files are the same +cat cool.json | jq -r '.data_array[] | "\(.CHANNEL_ID) \(.TileCalibBlob)"' > cool.txt +cat crest.json | jq -r '.data| to_entries| .[]| "\(.key) \(.value[0])"' > crest.txt + +cat cool.txt | sort -k 1 > cool-sorted.txt +cat crest.txt | sort -k 1 > crest-sorted.txt + +echo "Comparing CREST and COOL content" +diff cool-sorted.txt crest-sorted.txt +if [ $? -ne 0 ]; then + echo "CREST and COOL content are different" +else + echo "CREST and COOL content are the same" +fi +echo "clean up (only json files are kept)" +rm cool.txt crest.txt cool-sorted.txt crest-sorted.txt diff --git a/crestdb-client/curl/crest-test.sh b/crestdb-client/curl/crest-test.sh index 84d7b491..300167b8 100755 --- a/crestdb-client/curl/crest-test.sh +++ b/crestdb-client/curl/crest-test.sh @@ -20,6 +20,20 @@ generate_globaltag_data() EOF } +generate_tagmeta_data() +{ + cat < " echo "Use commands: get_data, create_tag, multi_upload.. " echo "get_data: " echo "multi_upload: " echo "create_tag: " + echo "create_tagmeta: " echo "create_globaltag: " echo "map_tag_to_gtag: