From 7e07dbf521e4004dc6683e3821c23e35d555e6ce Mon Sep 17 00:00:00 2001 From: Quentin Ligier Date: Mon, 23 Feb 2026 19:55:36 +0100 Subject: [PATCH 1/3] Add custom metrics to micrometer --- .../ca/uhn/fhir/jpa/starter/Application.java | 4 +- .../config/MatchboxMetricsConfig.java | 47 +++++++++++++++++++ .../StructureMapTransformProvider.java | 10 ++-- .../ImplementationGuideProviderR4.java | 5 ++ .../ImplementationGuideProviderR4B.java | 5 ++ .../ImplementationGuideProviderR5.java | 5 ++ .../MatchboxImplementationGuideProvider.java | 5 ++ .../matchbox/util/EngineSessionCache.java | 7 +++ .../matchbox/util/MatchboxEngineSupport.java | 4 ++ .../util/metrics/MatchboxMetrics.java | 47 +++++++++++++++++++ .../validation/ValidationProvider.java | 13 +++-- .../gazelle/GazelleValidationWs.java | 9 +++- 12 files changed, 149 insertions(+), 12 deletions(-) create mode 100644 matchbox-server/src/main/java/ch/ahdis/matchbox/config/MatchboxMetricsConfig.java create mode 100644 matchbox-server/src/main/java/ch/ahdis/matchbox/util/metrics/MatchboxMetrics.java diff --git a/matchbox-server/src/main/java/ca/uhn/fhir/jpa/starter/Application.java b/matchbox-server/src/main/java/ca/uhn/fhir/jpa/starter/Application.java index 471e4f2db04..0e7d91a5bdb 100644 --- a/matchbox-server/src/main/java/ca/uhn/fhir/jpa/starter/Application.java +++ b/matchbox-server/src/main/java/ca/uhn/fhir/jpa/starter/Application.java @@ -1,6 +1,7 @@ package ca.uhn.fhir.jpa.starter; import ch.ahdis.matchbox.MatchboxRestfulServer; +import ch.ahdis.matchbox.config.MatchboxMetricsConfig; import ch.ahdis.matchbox.config.MatchboxStaticResourceConfig; import ch.ahdis.matchbox.config.MatchboxTxConfig; import ch.ahdis.matchbox.spring.MatchboxEventListener; @@ -39,7 +40,8 @@ RegistryWs.class, MatchboxStaticResourceConfig.class, McpServerConfig.class, - MatchboxTxConfig.class + MatchboxTxConfig.class, + MatchboxMetricsConfig.class }) public class Application extends SpringBootServletInitializer { diff --git a/matchbox-server/src/main/java/ch/ahdis/matchbox/config/MatchboxMetricsConfig.java b/matchbox-server/src/main/java/ch/ahdis/matchbox/config/MatchboxMetricsConfig.java new file mode 100644 index 00000000000..2df3b2de52d --- /dev/null +++ b/matchbox-server/src/main/java/ch/ahdis/matchbox/config/MatchboxMetricsConfig.java @@ -0,0 +1,47 @@ +package ch.ahdis.matchbox.config; + +import ch.ahdis.matchbox.packages.MatchboxImplementationGuideProvider; +import ch.ahdis.matchbox.util.EngineSessionCache; +import ch.ahdis.matchbox.util.MatchboxEngineSupport; +import ch.ahdis.matchbox.util.metrics.MatchboxMetrics; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.binder.MeterBinder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Configuration for collecting Matchbox metrics and exposing them via Micrometer. + **/ +@Configuration +public class MatchboxMetricsConfig { + private static final String ENGINE_UNIT = "engines"; + + @Bean + public MeterBinder exposeNumberOfCachedEngines(final MatchboxEngineSupport engineSupport) { + final var engineCache = engineSupport.getSessionCache(); + return registry -> { + Gauge.builder("matchbox.engines.cached.transient.number", engineCache::numberOfTransientEngines) + .description("Number of cached expiring Matchbox engines in the server") + .baseUnit(ENGINE_UNIT) + .register(registry); + Gauge.builder("matchbox.engines.cached.permanent.number", engineCache::numberOfPermanentEngines) + .description("Number of cached immutable Matchbox engines in the server") + .baseUnit(ENGINE_UNIT) + .register(registry); + }; + } + + @Bean + public MeterBinder exposeNumberOfIgs(final MatchboxImplementationGuideProvider implementationGuideProvider) { + return registry -> Gauge.builder("matchbox.igs.number", implementationGuideProvider::count) + .description("Number of installed ImplementationGuides") + .baseUnit("ImplementationGuides") + .register(registry); + } + + @Bean + public MatchboxMetrics matchboxMetrics(final MeterRegistry meterRegistry) { + return new MatchboxMetrics(meterRegistry); + } +} diff --git a/matchbox-server/src/main/java/ch/ahdis/matchbox/mappinglanguage/StructureMapTransformProvider.java b/matchbox-server/src/main/java/ch/ahdis/matchbox/mappinglanguage/StructureMapTransformProvider.java index 290356e6761..792b5f5c515 100644 --- a/matchbox-server/src/main/java/ch/ahdis/matchbox/mappinglanguage/StructureMapTransformProvider.java +++ b/matchbox-server/src/main/java/ch/ahdis/matchbox/mappinglanguage/StructureMapTransformProvider.java @@ -2,9 +2,9 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.Map; -import java.util.Set; +import java.util.*; +import ch.ahdis.matchbox.util.metrics.MatchboxMetrics; import jakarta.servlet.ServletOutputStream; /* * #%L @@ -59,8 +59,6 @@ import org.hl7.fhir.r5.model.*; import javax.annotation.Nullable; -import java.util.ArrayList; -import java.util.List; /** * StructureMapTransformProvider @@ -70,6 +68,9 @@ public class StructureMapTransformProvider extends StructureMapResourceProvider @Autowired protected MatchboxEngineSupport matchboxEngineSupport; + @Autowired(required = false) + private Optional matchboxMetrics; + private final FhirContext fhirR5Context = FhirContext.forR5Cached(); @Override @@ -97,6 +98,7 @@ public MethodOutcome update(final HttpServletRequest theRequest, @Operation(name = "$transform", type = StructureMap.class, manualResponse = true, manualRequest = true) public void manualInputAndOutput(final HttpServletRequest theServletRequest, final HttpServletResponse theServletResponse) throws IOException { + this.matchboxMetrics.ifPresent(MatchboxMetrics::addTransformation); // Parse the request body, it is either a Parameters resource, or any resource final String body = new String(theServletRequest.getInputStream().readAllBytes()).trim(); @Nullable String resource = null; diff --git a/matchbox-server/src/main/java/ch/ahdis/matchbox/packages/ImplementationGuideProviderR4.java b/matchbox-server/src/main/java/ch/ahdis/matchbox/packages/ImplementationGuideProviderR4.java index 3e626607197..893ce14b820 100644 --- a/matchbox-server/src/main/java/ch/ahdis/matchbox/packages/ImplementationGuideProviderR4.java +++ b/matchbox-server/src/main/java/ch/ahdis/matchbox/packages/ImplementationGuideProviderR4.java @@ -547,4 +547,9 @@ public void installFromInternetRegistry(final String packageId, final String pac .setVersion(packageVersion) ); } + + @Override + public long count() { + return this.myPackageVersionDao.count(); + } } diff --git a/matchbox-server/src/main/java/ch/ahdis/matchbox/packages/ImplementationGuideProviderR4B.java b/matchbox-server/src/main/java/ch/ahdis/matchbox/packages/ImplementationGuideProviderR4B.java index 16b4e7d37df..c55a484ee53 100644 --- a/matchbox-server/src/main/java/ch/ahdis/matchbox/packages/ImplementationGuideProviderR4B.java +++ b/matchbox-server/src/main/java/ch/ahdis/matchbox/packages/ImplementationGuideProviderR4B.java @@ -551,4 +551,9 @@ public void installFromInternetRegistry(final String packageId, final String pac .setVersion(packageVersion) ); } + + @Override + public long count() { + return this.myPackageVersionDao.count(); + } } diff --git a/matchbox-server/src/main/java/ch/ahdis/matchbox/packages/ImplementationGuideProviderR5.java b/matchbox-server/src/main/java/ch/ahdis/matchbox/packages/ImplementationGuideProviderR5.java index 6e0a7532ec2..90f4cc7754c 100644 --- a/matchbox-server/src/main/java/ch/ahdis/matchbox/packages/ImplementationGuideProviderR5.java +++ b/matchbox-server/src/main/java/ch/ahdis/matchbox/packages/ImplementationGuideProviderR5.java @@ -533,4 +533,9 @@ public void installFromInternetRegistry(final String packageId, final String pac .setVersion(packageVersion) ); } + + @Override + public long count() { + return this.myPackageVersionDao.count(); + } } diff --git a/matchbox-server/src/main/java/ch/ahdis/matchbox/packages/MatchboxImplementationGuideProvider.java b/matchbox-server/src/main/java/ch/ahdis/matchbox/packages/MatchboxImplementationGuideProvider.java index 03253153d77..710177160af 100644 --- a/matchbox-server/src/main/java/ch/ahdis/matchbox/packages/MatchboxImplementationGuideProvider.java +++ b/matchbox-server/src/main/java/ch/ahdis/matchbox/packages/MatchboxImplementationGuideProvider.java @@ -20,4 +20,9 @@ public interface MatchboxImplementationGuideProvider { * Installs the given ImplementationGuide from the internet registry. */ void installFromInternetRegistry(final String packageId, final String packageVersion); + + /** + * Counts the number of installed ImplementationGuides. + */ + long count(); } diff --git a/matchbox-server/src/main/java/ch/ahdis/matchbox/util/EngineSessionCache.java b/matchbox-server/src/main/java/ch/ahdis/matchbox/util/EngineSessionCache.java index c72b0e5b721..cba2888275e 100644 --- a/matchbox-server/src/main/java/ch/ahdis/matchbox/util/EngineSessionCache.java +++ b/matchbox-server/src/main/java/ch/ahdis/matchbox/util/EngineSessionCache.java @@ -121,4 +121,11 @@ public String getSessionId(ValidationEngine validationEngine) { return null; } + public int numberOfPermanentEngines() { + return cachedSessionsNoTimeout.size(); + } + + public int numberOfTransientEngines() { + return cachedSessionIds.size(); + } } diff --git a/matchbox-server/src/main/java/ch/ahdis/matchbox/util/MatchboxEngineSupport.java b/matchbox-server/src/main/java/ch/ahdis/matchbox/util/MatchboxEngineSupport.java index 64951056351..21be96e1972 100644 --- a/matchbox-server/src/main/java/ch/ahdis/matchbox/util/MatchboxEngineSupport.java +++ b/matchbox-server/src/main/java/ch/ahdis/matchbox/util/MatchboxEngineSupport.java @@ -477,6 +477,10 @@ public String getSessionId(final MatchboxEngine engine) { return this.sessionCache.getSessionId(engine); } + public EngineSessionCache getSessionCache() { + return this.sessionCache; + } + public boolean isInitialized() { return initialized; } diff --git a/matchbox-server/src/main/java/ch/ahdis/matchbox/util/metrics/MatchboxMetrics.java b/matchbox-server/src/main/java/ch/ahdis/matchbox/util/metrics/MatchboxMetrics.java new file mode 100644 index 00000000000..8da6960beae --- /dev/null +++ b/matchbox-server/src/main/java/ch/ahdis/matchbox/util/metrics/MatchboxMetrics.java @@ -0,0 +1,47 @@ +package ch.ahdis.matchbox.util.metrics; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; + +import java.time.Duration; + +/** + * The class holding different metrics that Matchbox may collect. + * + * @see ch.ahdis.matchbox.config.MatchboxMetricsConfig + **/ +public class MatchboxMetrics { + + private final Counter validationCounter; + private final Timer validationDurationTimer; + private final Counter transformationCounter; + + public MatchboxMetrics(final MeterRegistry meterRegistry) { + this.validationCounter = Counter.builder("matchbox.validation.count") + .description("Number of FHIR resources validated by Matchbox") + .baseUnit("validations") + .register(meterRegistry); + this.validationDurationTimer = Timer.builder("matchbox.validation.duration") + .description("Duration of FHIR resource validation by Matchbox") + .distributionStatisticExpiry(Duration.ofDays(30)) + .distributionStatisticBufferLength(1000) + .register(meterRegistry); + this.transformationCounter = Counter.builder("matchbox.transformation.count") + .description("Number of FHIR resources transformed by Matchbox") + .baseUnit("transformations") + .register(meterRegistry); + } + + public void addValidation() { + this.validationCounter.increment(); + } + + public void addValidationDuration(final Duration duration) { + this.validationDurationTimer.record(duration); + } + + public void addTransformation() { + this.transformationCounter.increment(); + } +} diff --git a/matchbox-server/src/main/java/ch/ahdis/matchbox/validation/ValidationProvider.java b/matchbox-server/src/main/java/ch/ahdis/matchbox/validation/ValidationProvider.java index 9c18b0d2896..e101b6cdf5d 100644 --- a/matchbox-server/src/main/java/ch/ahdis/matchbox/validation/ValidationProvider.java +++ b/matchbox-server/src/main/java/ch/ahdis/matchbox/validation/ValidationProvider.java @@ -29,6 +29,7 @@ import ch.ahdis.matchbox.CliContext; import ch.ahdis.matchbox.config.MatchboxFhirVersion; import ch.ahdis.matchbox.util.MatchboxEngineSupport; +import ch.ahdis.matchbox.util.metrics.MatchboxMetrics; import ch.ahdis.matchbox.validation.matchspark.LLMConnector; import ch.ahdis.matchbox.engine.MatchboxEngine; import ch.ahdis.matchbox.engine.cli.VersionUtil; @@ -38,14 +39,10 @@ import org.checkerframework.checker.nullness.qual.NonNull; import org.hl7.fhir.instance.model.api.IBase; import org.hl7.fhir.instance.model.api.IBaseResource; -import org.hl7.fhir.r5.model.CodeableConcept; +import org.hl7.fhir.r5.model.*; import org.hl7.fhir.r5.elementmodel.Manager.FhirFormat; import org.hl7.fhir.r5.extensions.ExtensionDefinitions; -import org.hl7.fhir.r5.model.Duration; -import org.hl7.fhir.r5.model.OperationOutcome; -import org.hl7.fhir.r5.model.StringType; import org.hl7.fhir.r5.utils.EOperationOutcome; -import org.hl7.fhir.r5.model.UriType; import org.hl7.fhir.r5.utils.OperationOutcomeUtilities; import org.hl7.fhir.utilities.validation.ValidationMessage; import org.springframework.beans.factory.annotation.Autowired; @@ -60,6 +57,7 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import static ch.ahdis.matchbox.util.MatchboxServerUtils.addExtension; @@ -90,6 +88,9 @@ public class ValidationProvider { @Autowired private PlatformTransactionManager myTxManager; + @Autowired(required = false) + private Optional matchboxMetrics; + // @Operation(name = "$canonical", manualRequest = true, idempotent = true, returnParameters = { // @OperationParam(name = "return", type = IBase.class, min = 1, max = 1) }) // public IBaseResource canonical(HttpServletRequest theRequest) { @@ -114,6 +115,7 @@ public class ValidationProvider { @OperationParam(name = "return", type = IBase.class, min = 1, max = 1)}) public IBaseResource validate(final HttpServletRequest theRequest) { log.debug("$validate"); + this.matchboxMetrics.ifPresent(MatchboxMetrics::addValidation); final var sw = new StopWatch(); sw.startTask("Total"); @@ -222,6 +224,7 @@ public IBaseResource validate(final HttpServletRequest theRequest) { long millis = sw.getMillis(); log.debug("Validation time: {}", sw); + this.matchboxMetrics.ifPresent(m -> m.addValidationDuration(java.time.Duration.ofMillis(millis))); var oo = this.getOperationOutcome(sha3Hex, messages, profile, engine, millis, cliContext); diff --git a/matchbox-server/src/main/java/ch/ahdis/matchbox/validation/gazelle/GazelleValidationWs.java b/matchbox-server/src/main/java/ch/ahdis/matchbox/validation/gazelle/GazelleValidationWs.java index b8be5d8e73d..9184a9b0f11 100644 --- a/matchbox-server/src/main/java/ch/ahdis/matchbox/validation/gazelle/GazelleValidationWs.java +++ b/matchbox-server/src/main/java/ch/ahdis/matchbox/validation/gazelle/GazelleValidationWs.java @@ -3,6 +3,7 @@ import ca.uhn.fhir.jpa.model.entity.NpmPackageVersionResourceEntity; import ca.uhn.fhir.rest.api.EncodingEnum; import ca.uhn.fhir.util.StopWatch; +import ch.ahdis.matchbox.util.metrics.MatchboxMetrics; import ch.ahdis.matchbox.validation.ValidationProvider; import ch.ahdis.matchbox.CliContext; import ch.ahdis.matchbox.util.MatchboxEngineSupport; @@ -29,6 +30,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.Optional; import static ch.ahdis.matchbox.packages.MatchboxJpaPackageCache.structureDefinitionIsValidatable; @@ -50,18 +52,20 @@ public class GazelleValidationWs { private static final String VALIDATE_PATH = "/validation/validate"; private final MatchboxEngineSupport matchboxEngineSupport; - private final StructureDefinitionResourceProvider structureDefinitionProvider; + private final Optional matchboxMetrics; // The base CLI context, with the default parameters private final CliContext baseCliContext; public GazelleValidationWs(final MatchboxEngineSupport matchboxEngineSupport, final CliContext baseCliContext, - final StructureDefinitionResourceProvider structureDefinitionProvider) { + final StructureDefinitionResourceProvider structureDefinitionProvider, + final Optional matchboxMetrics) { this.matchboxEngineSupport = Objects.requireNonNull(matchboxEngineSupport); this.baseCliContext = Objects.requireNonNull(baseCliContext); this.structureDefinitionProvider = Objects.requireNonNull(structureDefinitionProvider); + this.matchboxMetrics = Objects.requireNonNull(matchboxMetrics); } /** @@ -133,6 +137,7 @@ public List getProfiles() { @PostMapping(path = VALIDATE_PATH, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) public ValidationReport postValidate(@RequestBody final ValidationRequest validationRequest) { + this.matchboxMetrics.ifPresent(MatchboxMetrics::addValidation); final var sw = new StopWatch(); sw.startTask("Total"); From 36abded83d00e167aec1cf2b191640d413474d52 Mon Sep 17 00:00:00 2001 From: Quentin Ligier Date: Thu, 2 Apr 2026 17:23:24 +0200 Subject: [PATCH 2/3] Test reconfiguring the validation duration timer --- .../java/ch/ahdis/matchbox/util/metrics/MatchboxMetrics.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/matchbox-server/src/main/java/ch/ahdis/matchbox/util/metrics/MatchboxMetrics.java b/matchbox-server/src/main/java/ch/ahdis/matchbox/util/metrics/MatchboxMetrics.java index 8da6960beae..1fd4d7122aa 100644 --- a/matchbox-server/src/main/java/ch/ahdis/matchbox/util/metrics/MatchboxMetrics.java +++ b/matchbox-server/src/main/java/ch/ahdis/matchbox/util/metrics/MatchboxMetrics.java @@ -24,8 +24,8 @@ public MatchboxMetrics(final MeterRegistry meterRegistry) { .register(meterRegistry); this.validationDurationTimer = Timer.builder("matchbox.validation.duration") .description("Duration of FHIR resource validation by Matchbox") - .distributionStatisticExpiry(Duration.ofDays(30)) - .distributionStatisticBufferLength(1000) + .percentilePrecision(1) + .maximumExpectedValue(Duration.ofSeconds(60)) .register(meterRegistry); this.transformationCounter = Counter.builder("matchbox.transformation.count") .description("Number of FHIR resources transformed by Matchbox") From f6f10ba0a2ef31b08cefc25c2328ae6bb13d8a22 Mon Sep 17 00:00:00 2001 From: Quentin Ligier Date: Thu, 2 Apr 2026 17:42:29 +0200 Subject: [PATCH 3/3] Test memory changes --- .github/workflows/maven.yml | 7 ++++++- matchbox-engine/pom.xml | 6 +++++- matchbox-server/pom.xml | 30 ++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 451ca54c20f..0ed80c1d152 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -23,11 +23,16 @@ jobs: distribution: adopt cache: maven + - run: free -h + - run: grep -E 'MemTotal|MemFree|MemAvailable|SwapTotal|SwapFree' /proc/meminfo + - run: vmstat -s + - run: java -XshowSettings:vm -version + - name: Run the tests in Maven timeout-minutes: 15 # We need a timeout here env: - MAVEN_OPTS: -Xmx12g -Xms4g -XX:MaxMetaspaceSize=1g + MAVEN_OPTS: -Xmx20g run: mvn --batch-mode --no-transfer-progress --update-snapshots verify - uses: actions/upload-artifact@v4 diff --git a/matchbox-engine/pom.xml b/matchbox-engine/pom.xml index a0258f0d448..ff82f52e6de 100644 --- a/matchbox-engine/pom.xml +++ b/matchbox-engine/pom.xml @@ -146,11 +146,12 @@ matchbox-engine + org.apache.maven.plugins maven-surefire-plugin - -Xmx12g + -Xmx20g true 1 @@ -178,6 +179,9 @@ maven-failsafe-plugin true + -Xmx20g + 1 + true diff --git a/matchbox-server/pom.xml b/matchbox-server/pom.xml index 1eab574fb20..624734282cf 100644 --- a/matchbox-server/pom.xml +++ b/matchbox-server/pom.xml @@ -303,12 +303,42 @@ + + + org.apache.maven.plugins + maven-surefire-plugin + + -Xmx20g + true + 1 + + + + default-test + test + + test + + + + + + org.apache.maven.plugins + maven-resources-plugin + + ${project.build.sourceEncoding} + + + org.apache.maven.plugins maven-failsafe-plugin true + -Xmx20g + 1 + true