diff --git a/.github/workflows/build.deployment_test.yml b/.github/workflows/build.deployment_test.yml index e7705bb0ed..a499df219e 100644 --- a/.github/workflows/build.deployment_test.yml +++ b/.github/workflows/build.deployment_test.yml @@ -8,7 +8,7 @@ on: jobs: deployment_test: - runs-on: self-hosted-k8s-x-small + runs-on: self-hosted-k8s-small container: image: us-central1-docker.pkg.dev/da-cn-shared/ghcr/digital-asset/decentralized-canton-sync-dev/docker/splice-test-ci:0.3.12 diff --git a/.github/workflows/monthly-schedule.yml b/.github/workflows/monthly-schedule.yml new file mode 100644 index 0000000000..b10f37d386 --- /dev/null +++ b/.github/workflows/monthly-schedule.yml @@ -0,0 +1,53 @@ +name: Monthly Schedule + +on: + workflow_dispatch: + inputs: + version: + description: "Splice version, e.g. 0.8" + required: true + type: string + + month: + description: "Month in YYYY-MM format, e.g. 2026-08" + required: true + type: string + + dry_run: + description: "Dry run only — do not modify Monday" + required: true + default: true + type: boolean + +permissions: + contents: read + +jobs: + create-schedule: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.13" + + - name: Create monthly schedule + env: + MONDAY_API_TOKEN: ${{ secrets.MONDAY_API_TOKEN }} + MONDAY_BOARD_ID: ${{ secrets.MONDAY_BOARD_ID }} + shell: bash + run: | + ARGS=( + "${{ inputs.version }}" + "${{ inputs.month }}" + ) + + if [[ "${{ inputs.dry_run }}" == "true" ]]; then + ARGS+=("--dry-run") + fi + + python3 scripts/monthly-schedule.py "${ARGS[@]}" diff --git a/LATEST_RELEASE b/LATEST_RELEASE index 0a1ffad4b4..8bd6ba8c5c 100644 --- a/LATEST_RELEASE +++ b/LATEST_RELEASE @@ -1 +1 @@ -0.7.4 +0.7.5 diff --git a/VERSION b/VERSION index 8bd6ba8c5c..a3df0a6959 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.7.5 +0.8.0 diff --git a/apps/app/src/main/scala/org/lfdecentralizedtrust/splice/console/ScanAppReference.scala b/apps/app/src/main/scala/org/lfdecentralizedtrust/splice/console/ScanAppReference.scala index fd9c5c4e3b..18eaf51dbf 100644 --- a/apps/app/src/main/scala/org/lfdecentralizedtrust/splice/console/ScanAppReference.scala +++ b/apps/app/src/main/scala/org/lfdecentralizedtrust/splice/console/ScanAppReference.scala @@ -27,11 +27,7 @@ import org.lfdecentralizedtrust.splice.codegen.java.splice.ans.AnsRules import org.lfdecentralizedtrust.splice.config.NetworkAppClientConfig import org.lfdecentralizedtrust.splice.environment.SpliceConsoleEnvironment import org.lfdecentralizedtrust.splice.http.v0.definitions -import org.lfdecentralizedtrust.splice.http.v0.definitions.{ - GetDsoInfoResponse, - UpdateHistoryItem, - UpdateHistoryItemV2, -} +import org.lfdecentralizedtrust.splice.http.v0.definitions.{UpdateHistoryItem, UpdateHistoryItemV2} import org.lfdecentralizedtrust.splice.scan.{ScanApp, ScanAppBootstrap} import org.lfdecentralizedtrust.splice.store.VoteResultsFilters import org.lfdecentralizedtrust.splice.scan.automation.ScanAutomationService @@ -43,6 +39,7 @@ import org.lfdecentralizedtrust.splice.util.{ ChoiceContextWithDisclosures, Contract, ContractWithState, + DsoInfo, FactoryChoiceWithDisclosures, PackageQualifiedName, SpliceUtil, @@ -89,7 +86,7 @@ abstract class ScanAppReference( httpCommand(HttpScanAppClient.GetDsoPartyId(List())) } - def getDsoInfo(): GetDsoInfoResponse = { + def getDsoInfo(): DsoInfo = { consoleEnvironment.run { httpCommand(HttpScanAppClient.GetDsoInfo(List())) } @@ -452,6 +449,31 @@ abstract class ScanAppReference( ) } + def getAcsSnapshotAtV2( + at: CantonTimestamp, + migrationId: Long, + recordTimeMatch: Option[definitions.AcsRequestV2.RecordTimeMatch] = Some( + definitions.AcsRequestV2.RecordTimeMatch.Exact + ), + after: Option[String] = None, + pageSize: Int = 100, + partyIds: Option[Vector[PartyId]] = None, + templates: Option[Vector[PackageQualifiedName]] = None, + ) = + consoleEnvironment.run { + httpCommand( + HttpScanAppClient.GetAcsSnapshotAtV2( + at.toInstant.atOffset(java.time.ZoneOffset.UTC), + migrationId, + recordTimeMatch, + after, + pageSize, + partyIds, + templates, + ) + ) + } + def getHoldingsStateAt( at: CantonTimestamp, migrationId: Long, diff --git a/apps/app/src/main/scala/org/lfdecentralizedtrust/splice/console/SvAppReference.scala b/apps/app/src/main/scala/org/lfdecentralizedtrust/splice/console/SvAppReference.scala index 9f9083697c..cf31943d33 100644 --- a/apps/app/src/main/scala/org/lfdecentralizedtrust/splice/console/SvAppReference.scala +++ b/apps/app/src/main/scala/org/lfdecentralizedtrust/splice/console/SvAppReference.scala @@ -42,7 +42,7 @@ import org.lfdecentralizedtrust.splice.sv.config.{ } import org.lfdecentralizedtrust.splice.sv.migration.SynchronizerNodeIdentities import org.lfdecentralizedtrust.splice.sv.util.ValidatorOnboarding -import org.lfdecentralizedtrust.splice.util.Contract +import org.lfdecentralizedtrust.splice.util.{Contract, DsoInfo} import java.time.Instant import scala.concurrent.duration.FiniteDuration @@ -89,9 +89,9 @@ abstract class SvAppReference( httpCommand(HttpSvPublicAppClient.DevNetOnboardValidatorPrepare()) } - def getDsoInfo(): HttpSvPublicAppClient.DsoInfo = + def getDsoInfo(): DsoInfo = consoleEnvironment.run { - httpCommand(HttpSvPublicAppClient.GetDsoInfo) + httpCommand(HttpSvOperatorAppClient.GetDsoInfo) } @Help.Summary("Get the CometBFT node status") @@ -292,13 +292,10 @@ class SvAppBackendReference( def appState: SvApp.State = _appState[SvApp.State, SvApp] @Help.Summary( - "Returns the current delegate based automation. Do not keep references to the result, as this automation gets replaced whenever the DSO delegate changes." + "Returns the delegate based automation. The reference is stable for the lifetime of the app." ) - def dsoDelegateBasedAutomation: DsoDelegateBasedAutomationService = { - appState.dsoAutomation.restartDsoDelegateBasedAutomationTrigger.epochState - .getOrElse(throw new RuntimeException("LeaderBasedAutomation is not fully started up")) - .dsoDelegateBasedAutomation - } + def dsoDelegateBasedAutomation: DsoDelegateBasedAutomationService = + appState.dsoAutomation.dsoDelegateBasedAutomation @Help.Summary( "Returns the current DSO automation." diff --git a/apps/app/src/main/scala/org/lfdecentralizedtrust/splice/console/ValidatorAppReference.scala b/apps/app/src/main/scala/org/lfdecentralizedtrust/splice/console/ValidatorAppReference.scala index 0fdfa9b317..dc5c159ef6 100644 --- a/apps/app/src/main/scala/org/lfdecentralizedtrust/splice/console/ValidatorAppReference.scala +++ b/apps/app/src/main/scala/org/lfdecentralizedtrust/splice/console/ValidatorAppReference.scala @@ -19,6 +19,7 @@ import org.lfdecentralizedtrust.splice.scan.admin.api.client.commands.HttpScanAp import org.lfdecentralizedtrust.splice.util.{ ChoiceContextWithDisclosures, ContractWithState, + DsoInfo, FactoryChoiceWithDisclosures, } import org.lfdecentralizedtrust.splice.validator.admin.api.client.commands.* @@ -360,7 +361,7 @@ abstract class ValidatorAppReference( } } - def getDsoInfo(): definitions.GetDsoInfoResponse = { + def getDsoInfo(): DsoInfo = { consoleEnvironment.run { httpCommand( HttpScanProxyAppClient.GetDsoInfo diff --git a/apps/app/src/pack/examples/sv-helm/sv-values.yaml b/apps/app/src/pack/examples/sv-helm/sv-values.yaml index 7e29f87695..7a7c46a89b 100644 --- a/apps/app/src/pack/examples/sv-helm/sv-values.yaml +++ b/apps/app/src/pack/examples/sv-helm/sv-values.yaml @@ -1,5 +1,6 @@ joinWithKeyOnboarding: sponsorApiUrl: "https://sv.sv-2.TARGET_HOSTNAME" + sponsorScanUrl: "https://scan.sv-2.TARGET_HOSTNAME" # Replace YOUR_SV_NAME with the name you provided for your SV identity onboardingName: YOUR_SV_NAME diff --git a/apps/app/src/test/resources/include/svs/sv2.conf b/apps/app/src/test/resources/include/svs/sv2.conf index 6abef88109..e60414a98d 100644 --- a/apps/app/src/test/resources/include/svs/sv2.conf +++ b/apps/app/src/test/resources/include/svs/sv2.conf @@ -25,6 +25,7 @@ url = "http://127.0.0.1:"${?canton.sv-apps.sv1.admin-api.port} url = ${?SV1_URL} } + include required("../scan-client") # keys generated using scripts/generate-sv-keys.sh include required("_sv2-id") private-key = "MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgYsPjtGcZLnQ3Ck35lm2805dL1Ds/JTKqlyGLbDewgEuhRANCAARV23y0sB+/7ofqzoQalgxu2FJ20RvKRQ7YVq7STbCKl//oLQD/7HMq2oomUGTJtwGIgIb9micaS4qBYEALWNUC" diff --git a/apps/app/src/test/resources/include/svs/sv3.conf b/apps/app/src/test/resources/include/svs/sv3.conf index 6c9fe7970f..d71a876762 100644 --- a/apps/app/src/test/resources/include/svs/sv3.conf +++ b/apps/app/src/test/resources/include/svs/sv3.conf @@ -25,6 +25,7 @@ url = "http://127.0.0.1:"${?canton.sv-apps.sv1.admin-api.port} url = ${?SV1_URL} } + include required("../scan-client") # keys generated using scripts/generate-sv-keys.sh include required("_sv3-id") private-key = "MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgWqaaIK3+vm8fymB1ZPh6PHYk/J7GyOYUdchj4I5gpzKhRANCAATuwdANiRWKxXOe4Wq+NZbEfe7xL5/Tt/UcJn6bH7KPbzKxEmNtq082exUBuIUO7Zc6rIhPH6iz8Z3f2mI5/LDb" diff --git a/apps/app/src/test/resources/include/svs/sv4.conf b/apps/app/src/test/resources/include/svs/sv4.conf index 4f44ecf45d..dac2fea28b 100644 --- a/apps/app/src/test/resources/include/svs/sv4.conf +++ b/apps/app/src/test/resources/include/svs/sv4.conf @@ -25,6 +25,7 @@ url = "http://127.0.0.1:"${?canton.sv-apps.sv1.admin-api.port} url = ${?SV1_URL} } + include required("../scan-client") # keys generated using scripts/generate-sv-keys.sh include required("_sv4-id") private-key = "MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgxED/gH8AeSwNujZAVLhBRSN55Hx0ntC6FKKhgn+7h92hRANCAARkw2wMmvW5PAxMgiXNRmlR7FMupUYywPtxHhjyyphgViGV1Ux4cbnNK5t/6n5ZlssTIxQJPmcEIIGHSiJRj1ys" diff --git a/apps/app/src/test/resources/preflight-topology.conf b/apps/app/src/test/resources/preflight-topology.conf index 73f326fbd7..aa0cf99ed9 100644 --- a/apps/app/src/test/resources/preflight-topology.conf +++ b/apps/app/src/test/resources/preflight-topology.conf @@ -62,6 +62,12 @@ canton { sv2Scan { admin-api.url = "https://scan.sv-2-eng."${NETWORK_APPS_ADDRESS}"" } + sv3Scan { + admin-api.url = "https://scan.sv-3-eng."${NETWORK_APPS_ADDRESS}"" + } + svda1Scan { + admin-api.url = "https://scan.sv-1."${NETWORK_APPS_ADDRESS}"" + } } splitwell-app-clients { diff --git a/apps/app/src/test/resources/sv-preflight-topology.conf b/apps/app/src/test/resources/sv-preflight-topology.conf index 524560ac59..6580472848 100644 --- a/apps/app/src/test/resources/sv-preflight-topology.conf +++ b/apps/app/src/test/resources/sv-preflight-topology.conf @@ -7,7 +7,7 @@ canton { } } scan-app-clients { - svTestScan { + svScan { admin-api { url = "https://scan.sv."${NETWORK_APPS_ADDRESS}"" } diff --git a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/plugins/EventHistorySanityCheckPlugin.scala b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/plugins/EventHistorySanityCheckPlugin.scala index c985cb6683..8508269f3b 100644 --- a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/plugins/EventHistorySanityCheckPlugin.scala +++ b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/plugins/EventHistorySanityCheckPlugin.scala @@ -88,7 +88,8 @@ class EventHistorySanityCheckPlugin( .fromJson(exercised.choiceArgument.noSpaces) .newSvParty == otherScan .getDsoInfo() - .svPartyId + .svParty + .toProtoPrimitive case _ => false }) } diff --git a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/plugins/UpdateHistorySanityCheckPlugin.scala b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/plugins/UpdateHistorySanityCheckPlugin.scala index d162d1df9f..d2c67504bf 100644 --- a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/plugins/UpdateHistorySanityCheckPlugin.scala +++ b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/plugins/UpdateHistorySanityCheckPlugin.scala @@ -5,7 +5,7 @@ import org.lfdecentralizedtrust.splice.config.ConfigTransforms.updateAllScanAppC import org.lfdecentralizedtrust.splice.config.SpliceConfig import org.lfdecentralizedtrust.splice.console.ScanAppBackendReference import org.lfdecentralizedtrust.splice.http.v0.definitions.DamlValueEncoding.members.CompactJson -import org.lfdecentralizedtrust.splice.http.v0.definitions.{AcsResponseV1, UpdateHistoryItemV2} +import org.lfdecentralizedtrust.splice.http.v0.definitions.{AcsResponseV2, UpdateHistoryItemV2} import org.lfdecentralizedtrust.splice.http.v0.definitions.UpdateHistoryItemV2.members import org.lfdecentralizedtrust.splice.http.v0.definitions.UpdateHistoryReassignment.Event.members as reassignmentMembers import org.lfdecentralizedtrust.splice.integration.tests.SpliceTests.SpliceTestConsoleEnvironment @@ -15,7 +15,6 @@ import com.digitalasset.canton.ScalaFuturesWithPatience import com.digitalasset.canton.data.CantonTimestamp import com.digitalasset.canton.logging.SuppressingLogger import com.digitalasset.canton.tracing.TraceContext -import org.lfdecentralizedtrust.splice.codegen.java.splice.dsorules.DsoRules import org.lfdecentralizedtrust.splice.scan.config.ScanStorageConfigs.scanStorageConfigV1 import org.lfdecentralizedtrust.splice.store.UpdateHistory.BackfillingState import org.scalatest.{Inspectors, LoneElement} @@ -65,10 +64,9 @@ class UpdateHistorySanityCheckPlugin( initializedScans.foreach(waitUntilBackfillingComplete) val (founders, others) = initializedScans.partition(_.config.isFirstSv) val founder = founders.loneElement - val dsoRules = - DsoRules.fromJson(founder.getDsoInfo().dsoRules.contract.payload.noSpaces) + val dsoRules = founder.getDsoInfo().dsoRules.payload val (scansInDsoRules, scansNotInDsoRules) = others.partition { otherScan => - val svPartyId = otherScan.getDsoInfo().svPartyId + val svPartyId = otherScan.getDsoInfo().svParty.toProtoPrimitive dsoRules.svs.containsKey(svPartyId) } scansNotInDsoRules.foreach { notInDso => @@ -78,7 +76,7 @@ class UpdateHistorySanityCheckPlugin( // - U2: Involves SV // founder sees U1, U2 but otherScan only U2 logger.info( - s"The SV party of Scan ${notInDso.name} (partyId=${notInDso.getDsoInfo().svPartyId}) is not in DsoRules. Ignoring." + s"The SV party of Scan ${notInDso.name} (partyId=${notInDso.getDsoInfo().svParty}) is not in DsoRules. Ignoring." ) } compareHistories(founder, scansInDsoRules) @@ -217,14 +215,14 @@ class UpdateHistorySanityCheckPlugin( private def getAllSnapshots( scan: ScanAppBackendReference, before: CantonTimestamp, - acc: List[AcsResponseV1], - ): List[AcsResponseV1] = { + acc: List[AcsResponseV2], + ): List[AcsResponseV2] = { val acsSnapshotPeriodHours = scanStorageConfigV1.dbAcsSnapshotPeriodHours val migrationId = scan.getMigrationId() scan.getDateOfMostRecentSnapshotBefore(before, migrationId) match { case Some(snapshotDate) => val snapshot = scan - .getAcsSnapshotAtV1( + .getAcsSnapshotAtV2( CantonTimestamp.assertFromInstant(snapshotDate.toInstant), migrationId, pageSize = 1000, diff --git a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/AppUpgradeIntegrationTest.scala b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/AppUpgradeIntegrationTest.scala index e0e1c85eb4..c75b4c23b2 100644 --- a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/AppUpgradeIntegrationTest.scala +++ b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/AppUpgradeIntegrationTest.scala @@ -26,6 +26,7 @@ import org.lfdecentralizedtrust.splice.wallet.store.BalanceChangeTxLogEntry import com.digitalasset.canton.config.CantonRequireTypes.InstanceName import com.digitalasset.canton.data.CantonTimestamp import com.digitalasset.canton.logging.{NamedLoggerFactory, NamedLogging} +import com.digitalasset.canton.topology.PartyId import com.digitalasset.canton.topology.admin.grpc.TopologyStoreId import com.digitalasset.canton.topology.store.TimeQuery.HeadState import monocle.macros.syntax.lens.* @@ -160,7 +161,6 @@ class AppUpgradeIntegrationTest bobValidatorBackend.participantClient.upload_dar_unless_exists(splitwellDarPathV1) val sv2Wallet = wc("sv2Wallet") - val sv1Client = sv_client("sv1Client") val bob = onboardWalletUser(bobWalletClient, bobValidatorBackend) @@ -208,11 +208,14 @@ class AppUpgradeIntegrationTest clue("Testing some more transactions after 2 SVs upgraded") { sv2Wallet.tap(1003) sv2Wallet.balance().unlockedQty should be > BigDecimal(2000) - // p2p transfer between an upgraded validator (alice's) and a non-upgraded (sv-1's) + // p2p transfer between an upgraded validator (alice's) and a non-upgraded (sv-1's). + // Note: we cannot use sv1's (authenticated, current-version) /v1/dso to look up its + // party here, as sv1 is still running the old release at this point. + // TODO(DACH-NY/canton-network-internal#2106) clean this up once the old release is new enough p2pTransfer( bobValidatorWalletClient, sv1WalletClient, - sv1Client.getDsoInfo().svParty, + PartyId.tryFromProtoPrimitive(sv1WalletClient.userStatus().party), 501, ) sv1WalletClient.balance().unlockedQty should be > BigDecimal(400) @@ -299,7 +302,7 @@ class AppUpgradeIntegrationTest )( "observing AmuletRules with upgraded config", _ => { - val newAmuletRules = sv1Client.getDsoInfo().amuletRules + val newAmuletRules = sv1Backend.getDsoInfo().amuletRules val config = newAmuletRules.payload.configSchedule.initialValue config.packageConfig.amulet should endWith(".123") diff --git a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/ScanFrontendTimeBasedIntegrationTest.scala b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/ScanFrontendTimeBasedIntegrationTest.scala index 2b23ed58cf..48fa6f23d6 100644 --- a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/ScanFrontendTimeBasedIntegrationTest.scala +++ b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/ScanFrontendTimeBasedIntegrationTest.scala @@ -103,7 +103,7 @@ class ScanFrontendTimeBasedIntegrationTest } contract should be( Some( - dsoInfo.dsoRules.contract.payload.asObject + dsoInfo.dsoRules.contract.toHttp.payload.asObject .valueOrFail("This is definitely an object.") ) ) diff --git a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/ScanIntegrationTest.scala b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/ScanIntegrationTest.scala index 0bbcb18f67..04493aa242 100644 --- a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/ScanIntegrationTest.scala +++ b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/ScanIntegrationTest.scala @@ -23,7 +23,6 @@ import org.lfdecentralizedtrust.splice.integration.tests.SpliceTests.{ SpliceTestConsoleEnvironment, } import org.lfdecentralizedtrust.splice.scan.config.CantonBftPeerConfig -import org.lfdecentralizedtrust.splice.sv.admin.api.client.commands.HttpSvPublicAppClient import org.lfdecentralizedtrust.splice.sv.automation.delegatebased.{ AdvanceOpenMiningRoundTrigger, ExpireIssuingMiningRoundTrigger, @@ -102,46 +101,30 @@ class ScanIntegrationTest "return dso info same as the sv app" in { implicit env => val scan = sv1ScanBackend.getDsoInfo() - inside(sv1Backend.getDsoInfo()) { - case HttpSvPublicAppClient.DsoInfo( - svUser, - svParty, - dsoParty, - votingThreshold, - latestMiningRound, - amuletRules, - dsoRules, - svNodeStates, - _, - ) => - scan.svUser should be(svUser) - scan.svPartyId should be(svParty.toProtoPrimitive) - scan.dsoPartyId should be(dsoParty.toProtoPrimitive) - scan.votingThreshold should be(votingThreshold) - scan.latestMiningRound should be(latestMiningRound.toHttp) - scan.amuletRules should be(amuletRules.toHttp) - scan.dsoRules should be(dsoRules.toHttp) - scan.svNodeStates should be(svNodeStates.map(_._2.toHttp)) - } + val svDsoInfo = sv1Backend.getDsoInfo() + scan shouldBe svDsoInfo + val dsoParty = scan.dsoParty clue("Returns physical synchronizer id") { sv1ScanBackend.getActivePhysicalSynchronizerSerial() shouldBe NonNegativeInt.zero } // sanity checks - scan.dsoRules.contract.contractId should be( + scan.dsoRules.contract.contractId.contractId should be( sv1Backend.participantClient.ledger_api_extensions.acs .filterJava(DsoRules.COMPANION)(dsoParty) .loneElement .id .contractId ) - scan.amuletRules.contract.contractId should be( + scan.amuletRules.contract.contractId.contractId should be( sv1Backend.participantClient.ledger_api_extensions.acs .filterJava(AmuletRules.COMPANION)(dsoParty) .loneElement .id .contractId ) - scan.svNodeStates.map(_.contract.contractId) should be( + scan.svNodeStates.values.map( + _.contract.contractId.contractId + ) should contain theSameElementsAs ( sv1Backend.participantClient.ledger_api_extensions.acs .filterJava(SvNodeState.COMPANION)(dsoParty) .map(_.id.contractId) diff --git a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/ScanTimeBasedIntegrationTest.scala b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/ScanTimeBasedIntegrationTest.scala index eea9c57dd9..a38a4968a1 100644 --- a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/ScanTimeBasedIntegrationTest.scala +++ b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/ScanTimeBasedIntegrationTest.scala @@ -254,7 +254,7 @@ class ScanTimeBasedIntegrationTest .getDateOfFirstSnapshotAfter(CantonTimestamp.tryFromInstant(snapshot1.value.toInstant), 0) .value shouldBe snapshotAfter.value - val snapshotAfterData = sv1ScanBackend.getAcsSnapshotAtV1( + val snapshotAfterData = sv1ScanBackend.getAcsSnapshotAtV2( CantonTimestamp.assertFromInstant(snapshotAfter.value.toInstant), migrationId, templates = Some( @@ -271,10 +271,10 @@ class ScanTimeBasedIntegrationTest val atOrBefore = getLedgerTime // afOrBefore should return the same ACS snapshot as the exact time given by snapshotAfter - val snapshotAtOrBeforeAfterData = sv1ScanBackend.getAcsSnapshotAtV1( + val snapshotAtOrBeforeAfterData = sv1ScanBackend.getAcsSnapshotAtV2( CantonTimestamp.assertFromInstant(atOrBefore.toInstant), migrationId, - recordTimeMatch = Some(definitions.AcsRequest.RecordTimeMatch.AtOrBefore), + recordTimeMatch = Some(definitions.AcsRequestV2.RecordTimeMatch.AtOrBefore), templates = Some( Vector( PackageQualifiedName.fromJavaCodegenCompanion(Amulet.COMPANION), @@ -287,10 +287,10 @@ class ScanTimeBasedIntegrationTest snapshotAfterData shouldBe snapshotAtOrBeforeAfterData snapshotAtOrBeforeAfterData.value.recordTime shouldBe snapshotAfter.value - sv1ScanBackend.getAcsSnapshotAtV1( + sv1ScanBackend.getAcsSnapshotAtV2( CantonTimestamp.assertFromInstant(atOrBefore.toInstant), migrationId, - recordTimeMatch = Some(definitions.AcsRequest.RecordTimeMatch.Exact), + recordTimeMatch = Some(definitions.AcsRequestV2.RecordTimeMatch.Exact), templates = Some( Vector( PackageQualifiedName.fromJavaCodegenCompanion(Amulet.COMPANION), @@ -486,7 +486,7 @@ class ScanTimeBasedIntegrationTest // Compare bulk storage data to hot storage data from scan // TODO(#4788): for now, bulk storage still uses v0, so we use that here as well val acsAtMidnightFromScan = sv1ScanBackend - .getAcsSnapshotAtV1(CantonTimestamp.assertFromInstant(lastMidnight), 0) + .getAcsSnapshotAtV2(CantonTimestamp.assertFromInstant(lastMidnight), 0) .value .createdEvents val acsObjUrl = getSnapshotResponse.objectRefs.head.url diff --git a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/SvOnboardingConfigIntegrationTest.scala b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/SvOnboardingConfigIntegrationTest.scala index 450b319e2c..2596e1575d 100644 --- a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/SvOnboardingConfigIntegrationTest.scala +++ b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/SvOnboardingConfigIntegrationTest.scala @@ -49,10 +49,13 @@ class SvOnboardingConfigIntegrationTest "An onboarded SV can initialize even if its onboarding sponsor is down" in { implicit env => startAllSync( + // sv1's scan is required for sv2's initial onboarding (but not for restarts, see below) + sv1ScanBackend, sv1Backend, sv2Backend, ) - clue("Stopping SV1 and SV2") { + clue("Stopping SV1 (incl. its scan) and SV2") { + sv1ScanBackend.stop() sv1Backend.stop() sv2Backend.stop() } diff --git a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/SvOnboardingViaNonFoundingSvIntegrationTest.scala b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/SvOnboardingViaNonFoundingSvIntegrationTest.scala index 76029cc09f..d89044d135 100644 --- a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/SvOnboardingViaNonFoundingSvIntegrationTest.scala +++ b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/SvOnboardingViaNonFoundingSvIntegrationTest.scala @@ -13,6 +13,7 @@ import org.lfdecentralizedtrust.splice.config.ConfigTransforms.{ import org.lfdecentralizedtrust.splice.config.{ConfigTransforms, NetworkAppClientConfig} import org.lfdecentralizedtrust.splice.integration.EnvironmentDefinition import org.lfdecentralizedtrust.splice.integration.tests.SpliceTests.IntegrationTestWithIsolatedEnvironment +import org.lfdecentralizedtrust.splice.scan.config.ScanAppClientConfig import org.lfdecentralizedtrust.splice.sv.{LocalSynchronizerNode, SvAppClientConfig} import org.lfdecentralizedtrust.splice.sv.automation.singlesv.SvBftSequencerPeerOffboardingTrigger import org.lfdecentralizedtrust.splice.sv.automation.singlesv.offboarding.SvOffboardingSequencerTrigger @@ -55,6 +56,17 @@ class SvOnboardingViaNonFoundingSvIntegrationTest case node: JoinWithKey => bumpUrl(sv1ToSv2Bump, node.svClient.adminApi.url.toString()) case _ => throw new IllegalStateException("JoinWithKey configuration not found.") } + val sv2OnboardingScanClientUrl = + configuration + .svApps(InstanceName.tryCreate("sv2")) + .onboarding + .getOrElse( + throw new IllegalStateException("Onboarding configuration not found.") + ) match { + case node: JoinWithKey => + bumpUrl(sv1ToSv2Bump, node.scanClient.adminApi.url.toString()) + case _ => throw new IllegalStateException("JoinWithKey configuration not found.") + } val sv2BootstrapSequencerUrl = configuration .svApps(InstanceName.tryCreate("sv2")) @@ -76,6 +88,8 @@ class SvOnboardingViaNonFoundingSvIntegrationTest SvAppClientConfig(NetworkAppClientConfig(sv2OnboardingSvClientUrl)), node.publicKey, node.privateKey, + // fetch DSO info via the sponsor's (sv2's) scan + ScanAppClientConfig(NetworkAppClientConfig(sv2OnboardingScanClientUrl)), ) ) case _ => throw new IllegalStateException("JoinWithKey configuration not found.") diff --git a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/SvStateManagementIntegrationTest.scala b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/SvStateManagementIntegrationTest.scala index b48d282e58..1dd895639d 100644 --- a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/SvStateManagementIntegrationTest.scala +++ b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/SvStateManagementIntegrationTest.scala @@ -675,6 +675,8 @@ class SvStateManagementIntegrationTest extends SvIntegrationTestBase with Trigge "Vote requests expire" in { implicit env => clue("Initialize DSO with 2 SVs") { startAllSync( + // sv1's scan is required for sv2's onboarding + sv1ScanBackend, sv1Backend, sv2Backend, ) diff --git a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/TestTokenV2SettlementIntegrationTest.scala b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/TestTokenV2SettlementIntegrationTest.scala index 135645a921..8c99eb8ce3 100644 --- a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/TestTokenV2SettlementIntegrationTest.scala +++ b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/TestTokenV2SettlementIntegrationTest.scala @@ -157,6 +157,34 @@ class TestTokenV2SettlementIntegrationTest def ttAdminValidator(implicit env: SpliceTestConsoleEnvironment) = v("testTokenValidatorLocal") + private def withValidatorsInitializedForTest[T]( + f: () => T + )(implicit env: SpliceTestConsoleEnvironment): T = { + sv1ValidatorBackend.startSync() // hosts DSO + Seq( + aliceValidatorBackend, // hosts Alice + bobValidatorBackend, // hosts Bob + splitwellValidatorBackend, // hosts the venue party + ttAdminValidator, // hosts the ttadmin + ).foreach { validatorBackend => + validatorBackend.startSync() + validatorBackend.participantClient.upload_dar_unless_exists(tokenStandardV2TestDarPath) + validatorBackend.participantClient + .upload_dar_unless_exists(testTokenV2DarPath) + } + try { + f() + } finally { + Seq( + sv1ValidatorBackend, + aliceValidatorBackend, + bobValidatorBackend, + splitwellValidatorBackend, + ttAdminValidator, + ).foreach(_.stop()) + } + } + "TestTokenV2 should be settleable" in { implicit env => initDso() withCanton( @@ -168,649 +196,640 @@ class TestTokenV2SettlementIntegrationTest "EXTRA_PARTICIPANT_ADMIN_USER" -> ttAdminValidator.config.ledgerApiUser, "EXTRA_PARTICIPANT_DB" -> dbName, ) { - sv1ValidatorBackend.startSync() // hosts DSO - Seq( - aliceValidatorBackend, // hosts Alice - bobValidatorBackend, // hosts Bob - splitwellValidatorBackend, // hosts the venue party - ttAdminValidator, // hosts the ttadmin - ).foreach { validatorBackend => - validatorBackend.startSync() - validatorBackend.participantClient.upload_dar_unless_exists(tokenStandardV2TestDarPath) - validatorBackend.participantClient - .upload_dar_unless_exists(testTokenV2DarPath) - } - val aliceParty = onboardWalletUser(aliceWalletClient, aliceValidatorBackend) - val bobParty = onboardWalletUser(bobWalletClient, bobValidatorBackend) - val venueValidator = splitwellValidatorBackend - val venueParty = PartyId.tryFromProtoPrimitive(splitwellWalletClient.userStatus().party) - val ttAdminParty = ttAdminValidator.getValidatorPartyId() - val registry = new TestTokenV2Registry(ttAdminParty, ttAdminValidator) - - // Give alice some CC - aliceWalletClient.tap(1000) - val aliceCCBalanceBefore = eventually() { - val balance = aliceWalletClient.balance().unlockedQty - balance should be > BigDecimal(0) - balance - } - - // make venue and ttadmin featured app parties - splitwellWalletClient.selfGrantFeaturedAppRight() - aliceValidatorWalletLocalClient.selfGrantFeaturedAppRight() - // We need to make sure that after this block, the oldest active OpenMiningRound - // has an `openAt` time that is after the time the first OpenMiningRound was archived, - // otherwise the ingestion start approximation in - // `DbScanRewardsReferenceStore.lookupActiveOpenMiningRounds` might filter out the - // OpenMiningRound contract and AppActivityComputation won't ingest activity records. - clue("Advance rounds") { - advanceRoundsByOneTickViaAutomation() // archives round 0, open rounds are 1-3 - advanceRoundsByOneTickViaAutomation() // archives round 1, open rounds are 2-4 - advanceRoundsByOneTickViaAutomation() // archives round 2, open rounds are 3-5 - advanceRoundsByOneTickViaAutomation() // archives round 3, open rounds are 4-6 - } + withValidatorsInitializedForTest { () => + val aliceParty = onboardWalletUser(aliceWalletClient, aliceValidatorBackend) + val bobParty = onboardWalletUser(bobWalletClient, bobValidatorBackend) + val venueValidator = splitwellValidatorBackend + val venueParty = PartyId.tryFromProtoPrimitive(splitwellWalletClient.userStatus().party) + val ttAdminParty = ttAdminValidator.getValidatorPartyId() + val registry = new TestTokenV2Registry(ttAdminParty, ttAdminValidator) + + // Give alice some CC + aliceWalletClient.tap(1000) + val aliceCCBalanceBefore = eventually() { + val balance = aliceWalletClient.balance().unlockedQty + balance should be > BigDecimal(0) + balance + } - // Create BatchingUtilityV2 contracts for Alice and Bob - val batchingUtilityIds: Map[PartyId, BatchingUtility.ContractId] = - Map(aliceValidatorBackend -> aliceParty, bobValidatorBackend -> bobParty).map { - case (validatorBackend, party) => - party -> validatorBackend.participantClientWithAdminToken.ledger_api_extensions.commands - .submitWithResult( - userId = validatorBackend.config.ledgerApiUser, - actAs = Seq(party), - readAs = Seq(party), - update = BatchingUtilityV2.create(party.toProtoPrimitive), - ) - .contractId + // make venue and ttadmin featured app parties + splitwellWalletClient.selfGrantFeaturedAppRight() + aliceValidatorWalletLocalClient.selfGrantFeaturedAppRight() + // We need to make sure that after this block, the oldest active OpenMiningRound + // has an `openAt` time that is after the time the first OpenMiningRound was archived, + // otherwise the ingestion start approximation in + // `DbScanRewardsReferenceStore.lookupActiveOpenMiningRounds` might filter out the + // OpenMiningRound contract and AppActivityComputation won't ingest activity records. + clue("Advance rounds") { + advanceRoundsByOneTickViaAutomation() // archives round 0, open rounds are 1-3 + advanceRoundsByOneTickViaAutomation() // archives round 1, open rounds are 2-4 + advanceRoundsByOneTickViaAutomation() // archives round 2, open rounds are 3-5 + advanceRoundsByOneTickViaAutomation() // archives round 3, open rounds are 4-6 } - // Create TokenRules for ttadmin - val tokenRulesId = ttAdminValidator.participantClient.ledger_api_extensions.commands - .submitWithResult( - userId = ttAdminValidator.config.ledgerApiUser, - actAs = Seq(ttAdminParty), - readAs = Seq(ttAdminParty), - update = TokenV2Rules.create(ttAdminParty.toProtoPrimitive), - ) - .contractId - - // Call TokenRules_OfferMint to offer 100 USDC to Bob - val bobConfigAccount = new testtokenv2.accountconfig.AccountConfig( - ttAdminParty.toProtoPrimitive, - basicAccount(bobParty), - new testtokenv2.accountconfig.PartyConfig(true, true), - new testtokenv2.accountconfig.PartyConfig(false, false), - ) - val bobOfferMintAmount = 100 - ttAdminValidator.participantClient.ledger_api_extensions.commands - .submitJava( - userId = ttAdminValidator.config.ledgerApiUser, - actAs = Seq(ttAdminParty), - commands = tokenRulesId - .exerciseTokenRules_OfferMint( - basicAccount(bobParty), - BigDecimal(bobOfferMintAmount).bigDecimal, - new holdingv2.InstrumentId(ttAdminParty.toProtoPrimitive, "USDC"), - Instant.now(), - bobConfigAccount, - ) - .commands() - .asScala - .toSeq, - ) + // Create BatchingUtilityV2 contracts for Alice and Bob + val batchingUtilityIds: Map[PartyId, BatchingUtility.ContractId] = + Map(aliceValidatorBackend -> aliceParty, bobValidatorBackend -> bobParty).map { + case (validatorBackend, party) => + party -> validatorBackend.participantClientWithAdminToken.ledger_api_extensions.commands + .submitWithResult( + userId = validatorBackend.config.ledgerApiUser, + actAs = Seq(party), + readAs = Seq(party), + update = BatchingUtilityV2.create(party.toProtoPrimitive), + ) + .contractId + } - // Bob accepts - val transferInstruction = eventually() { - Contract - .fromCreatedEvent(transferinstructionv2.TransferInstruction.INTERFACE)( - CreatedEvent.fromProto( - createdEventToJavaProto( - bobValidatorBackend.participantClientWithAdminToken.ledger_api.state.acs - .of_party( - party = bobParty, - filterInterfaces = - Seq(transferinstructionv2.TransferInstruction.TEMPLATE_ID).map(templateId => - TemplateId( - templateId.getPackageId, - templateId.getModuleName, - templateId.getEntityName, - ) - ), - ) - .loneElement - .event - ) - ) + // Create TokenRules for ttadmin + val tokenRulesId = ttAdminValidator.participantClient.ledger_api_extensions.commands + .submitWithResult( + userId = ttAdminValidator.config.ledgerApiUser, + actAs = Seq(ttAdminParty), + readAs = Seq(ttAdminParty), + update = TokenV2Rules.create(ttAdminParty.toProtoPrimitive), ) - .valueOrFail("Failed to read transferinstructionv2.TransferInstruction") - } - val acceptContext = - registry.getContext( - transferInstruction.payload.transfer.inputHoldingCids.asScala.toSeq + .contractId + + // Call TokenRules_OfferMint to offer 100 USDC to Bob + val bobConfigAccount = new testtokenv2.accountconfig.AccountConfig( + ttAdminParty.toProtoPrimitive, + basicAccount(bobParty), + new testtokenv2.accountconfig.PartyConfig(true, true), + new testtokenv2.accountconfig.PartyConfig(false, false), ) - val transferResult = - bobValidatorBackend.participantClientWithAdminToken.ledger_api_extensions.commands - .submitWithResult( - userId = bobValidatorBackend.config.ledgerApiUser, - actAs = Seq(bobParty), - readAs = Seq(bobParty), - update = transferInstruction.contractId.exerciseTransferInstruction_Accept( - java.util.List.of(bobParty.toProtoPrimitive), - new metadatav1.ExtraArgs(acceptContext.choiceContext, emptyMetadata), - ), - disclosedContracts = acceptContext.disclosedContracts, + val bobOfferMintAmount = 100 + ttAdminValidator.participantClient.ledger_api_extensions.commands + .submitJava( + userId = ttAdminValidator.config.ledgerApiUser, + actAs = Seq(ttAdminParty), + commands = tokenRulesId + .exerciseTokenRules_OfferMint( + basicAccount(bobParty), + BigDecimal(bobOfferMintAmount).bigDecimal, + new holdingv2.InstrumentId(ttAdminParty.toProtoPrimitive, "USDC"), + Instant.now(), + bobConfigAccount, + ) + .commands() + .asScala + .toSeq, ) - transferResult.exerciseResult.output match { - case completed: TransferInstructionResult_Completed => - completed.receiverHoldingCids.asScala.toSeq - case other => fail(s"Offer mint was not completed: $other") - } - // Venue creates the trade - val (createTradeTx, otcTrade) = actAndCheck( - "Venue creates OTC Trade", { - venueValidator.participantClientWithAdminToken.ledger_api_extensions.commands - .submitJava( - actAs = Seq(venueParty), - commands = new tradingappv2.OTCTrade( - venueParty.toProtoPrimitive, - Seq( - // Alice -> Bob: 100 CC - new tradingappv2.TradeLeg( - dsoParty.toProtoPrimitive, - new allocationv2.TransferLeg( - "alicetobob100CC", - basicAccount(aliceParty), - basicAccount(bobParty), - BigDecimal(100).bigDecimal, - amuletInstrumentIdName, - emptyMetadata, - ), - ), - // Bob -> Alice: 15 USDC - new tradingappv2.TradeLeg( - ttAdminParty.toProtoPrimitive, - new allocationv2.TransferLeg( - "bobtoalice15USDC", - basicAccount(bobParty), - basicAccount(aliceParty), - BigDecimal(15).bigDecimal, - usdcInstrumentName, - emptyMetadata, - ), - ), - // Alice -> Venue: 0.2 USDC - new tradingappv2.TradeLeg( - ttAdminParty.toProtoPrimitive, - new allocationv2.TransferLeg( - "alicetovenue0.2USDC", - basicAccount(aliceParty), - basicAccount(venueParty), - BigDecimal(0.2).bigDecimal, - usdcInstrumentName, - emptyMetadata, - ), - ), - ).asJava, - Instant.now(), - Instant.now().plusSeconds(60L), - java.util.Optional.of(Instant.now().plusSeconds(180L)), + // Bob accepts + val transferInstruction = eventually() { + Contract + .fromCreatedEvent(transferinstructionv2.TransferInstruction.INTERFACE)( + CreatedEvent.fromProto( + createdEventToJavaProto( + bobValidatorBackend.participantClientWithAdminToken.ledger_api.state.acs + .of_party( + party = bobParty, + filterInterfaces = + Seq(transferinstructionv2.TransferInstruction.TEMPLATE_ID).map(templateId => + TemplateId( + templateId.getPackageId, + templateId.getModuleName, + templateId.getEntityName, + ) + ), + ) + .loneElement + .event + ) ) - .create() - .commands() - .asScala - .toSeq, ) - }, - )( - "There exists a trade visible to the venue's participant", - _ => - venueValidator.participantClientWithAdminToken.ledger_api_extensions.acs - .awaitJava(tradingappv2.OTCTrade.COMPANION)( - venueParty - ), - ) + .valueOrFail("Failed to read transferinstructionv2.TransferInstruction") + } + val acceptContext = + registry.getContext( + transferInstruction.payload.transfer.inputHoldingCids.asScala.toSeq + ) + val transferResult = + bobValidatorBackend.participantClientWithAdminToken.ledger_api_extensions.commands + .submitWithResult( + userId = bobValidatorBackend.config.ledgerApiUser, + actAs = Seq(bobParty), + readAs = Seq(bobParty), + update = transferInstruction.contractId.exerciseTransferInstruction_Accept( + java.util.List.of(bobParty.toProtoPrimitive), + new metadatav1.ExtraArgs(acceptContext.choiceContext, emptyMetadata), + ), + disclosedContracts = acceptContext.disclosedContracts, + ) + transferResult.exerciseResult.output match { + case completed: TransferInstructionResult_Completed => + completed.receiverHoldingCids.asScala.toSeq + case other => fail(s"Offer mint was not completed: $other") + } - val (createAllocationRequestsTx, (bobAllocationRequest, aliceAllocationRequest)) = - actAndCheck( - "Venue creates allocation requests", { + // Venue creates the trade + val (createTradeTx, otcTrade) = actAndCheck( + "Venue creates OTC Trade", { venueValidator.participantClientWithAdminToken.ledger_api_extensions.commands .submitJava( actAs = Seq(venueParty), - commands = otcTrade.id - .exerciseOTCTrade_RequestAllocations() + commands = new tradingappv2.OTCTrade( + venueParty.toProtoPrimitive, + Seq( + // Alice -> Bob: 100 CC + new tradingappv2.TradeLeg( + dsoParty.toProtoPrimitive, + new allocationv2.TransferLeg( + "alicetobob100CC", + basicAccount(aliceParty), + basicAccount(bobParty), + BigDecimal(100).bigDecimal, + amuletInstrumentIdName, + emptyMetadata, + ), + ), + // Bob -> Alice: 15 USDC + new tradingappv2.TradeLeg( + ttAdminParty.toProtoPrimitive, + new allocationv2.TransferLeg( + "bobtoalice15USDC", + basicAccount(bobParty), + basicAccount(aliceParty), + BigDecimal(15).bigDecimal, + usdcInstrumentName, + emptyMetadata, + ), + ), + // Alice -> Venue: 0.2 USDC + new tradingappv2.TradeLeg( + ttAdminParty.toProtoPrimitive, + new allocationv2.TransferLeg( + "alicetovenue0.2USDC", + basicAccount(aliceParty), + basicAccount(venueParty), + BigDecimal(0.2).bigDecimal, + usdcInstrumentName, + emptyMetadata, + ), + ), + ).asJava, + Instant.now(), + Instant.now().plusSeconds(60L), + java.util.Optional.of(Instant.now().plusSeconds(180L)), + ) + .create() .commands() .asScala .toSeq, ) }, )( - "Sender and receiver see the allocation requests", - _ => { - val bobAllocationRequest = inside( - bobWalletClient.listAllocationRequests() - ) { - case (allocationRequest: HttpWalletAppClient.TokenStandard.V2AllocationRequest) +: Nil => - allocationRequest - } - val aliceAllocationRequest = inside( - aliceWalletClient.listAllocationRequests() - ) { - case (allocationRequest: HttpWalletAppClient.TokenStandard.V2AllocationRequest) +: Nil => - allocationRequest - } - - (bobAllocationRequest, aliceAllocationRequest) - }, + "There exists a trade visible to the venue's participant", + _ => + venueValidator.participantClientWithAdminToken.ledger_api_extensions.acs + .awaitJava(tradingappv2.OTCTrade.COMPANION)( + venueParty + ), ) - val (aliceAllocationCids, aliceAllocateTx) = clue( - "Alice uses the BatchingUtilityV2 to create two allocations and accept the allocation request in a single tx" - ) { - // UpdateExternalPartyConfigStateTrigger might run concurrently and cause a LOCAL_VERDICT_INACTIVE_CONTRACTS - // because of the ExternalPartyConfigState being updated. - // In the real world, we expect the venue to also just retry re-fetching all contexts - eventuallySucceeds() { - val batchingUtility = batchingUtilityIds(aliceParty) - val aliceAmulets = aliceWalletClient - .list() - .amulets - .map(_.contract.contractId.toInterface(holdingv2.Holding.INTERFACE)) - val amuletSpec = aliceAllocationRequest.contract.payload.allocations.asScala - .filter(_.admin == dsoParty.toProtoPrimitive) - .loneElement - val usdcSpec = aliceAllocationRequest.contract.payload.allocations.asScala - .filter(_.admin == ttAdminParty.toProtoPrimitive) - .loneElement - val amuletAllocationFactory = sv1ScanBackend.getAllocationFactoryV2( - new allocationinstructionv2.AllocationFactory_Allocate( - aliceAllocationRequest.contract.payload.settlement, - amuletSpec, - aliceAllocationRequest.contract.payload.requestedAt, - aliceAmulets.asJava, - emptyExtraArgs, - java.util.List.of(aliceParty.toProtoPrimitive), - ) + val (createAllocationRequestsTx, (bobAllocationRequest, aliceAllocationRequest)) = + actAndCheck( + "Venue creates allocation requests", { + venueValidator.participantClientWithAdminToken.ledger_api_extensions.commands + .submitJava( + actAs = Seq(venueParty), + commands = otcTrade.id + .exerciseOTCTrade_RequestAllocations() + .commands() + .asScala + .toSeq, + ) + }, + )( + "Sender and receiver see the allocation requests", + _ => { + val bobAllocationRequest = inside( + bobWalletClient.listAllocationRequests() + ) { + case (allocationRequest: HttpWalletAppClient.TokenStandard.V2AllocationRequest) +: Nil => + allocationRequest + } + val aliceAllocationRequest = inside( + aliceWalletClient.listAllocationRequests() + ) { + case (allocationRequest: HttpWalletAppClient.TokenStandard.V2AllocationRequest) +: Nil => + allocationRequest + } + + (bobAllocationRequest, aliceAllocationRequest) + }, ) - val usdcContext = registry.getContext(Seq.empty) - val aliceAllocateUpdate = batchingUtility - .exerciseBatchingUtility_ExecuteBatch( - new HoldingMap( - Map( - new ScopedAccount( - dsoParty.toProtoPrimitive, - basicAccount(aliceParty), - ) -> Map[String, java.util.List[holdingv2.Holding.ContractId]]( - amuletInstrumentIdName -> aliceAmulets.asJava - ).asJava, - new ScopedAccount( - ttAdminParty.toProtoPrimitive, - basicAccount(aliceParty), - ) -> Map - .empty[String, java.util.List[holdingv2.Holding.ContractId]] - .asJava, // alice has no USDC here yet - ).asJava - ), - java.util.List.of( - new TSA_AllocationFactory_AllocateV2( - new ChoiceCall[AllocationFactory_Allocate]( - new metadatav1.AnyContract.ContractId( - amuletAllocationFactory.factoryId.contractId - ), - amuletAllocationFactory.args, - ) - ), - new TSA_AllocationFactory_AllocateV2( - new ChoiceCall[AllocationFactory_Allocate]( - new metadatav1.AnyContract.ContractId(tokenRulesId.contractId), - new allocationinstructionv2.AllocationFactory_Allocate( - aliceAllocationRequest.contract.payload.settlement, - usdcSpec, - aliceAllocationRequest.contract.payload.requestedAt, - java.util.List.of(), - new metadatav1.ExtraArgs(usdcContext.choiceContext, emptyMetadata), - java.util.List.of(aliceParty.toProtoPrimitive), - ), - ) - ), - new TSA_AllocationRequest_AcceptV2( - new ChoiceCall[AllocationRequest_Accept]( - new metadatav1.AnyContract.ContractId( - aliceAllocationRequest.contract.contractId.contractId - ), - new AllocationRequest_Accept( - java.util.List.of(aliceParty.toProtoPrimitive), - amuletAllocationFactory.args.extraArgs, - ), - ) - ), - ), - true, - ) - val aliceAllocateTx = - aliceValidatorBackend.participantClientWithAdminToken.ledger_api_extensions.commands - .submitJava( - userId = aliceValidatorBackend.config.ledgerApiUser, - actAs = Seq(aliceParty), - readAs = Seq(aliceParty), - commands = aliceAllocateUpdate.commands().asScala.toSeq, - disclosedContracts = - amuletAllocationFactory.disclosedContracts ++ usdcContext.disclosedContracts, - ) - val aliceAllocationCids = SpliceLedgerConnection - .decodeExerciseResult( - aliceAllocateUpdate, - aliceAllocateTx, - ) - .exerciseResult - .actionResults - .asScala - .map { - case _: TSAR_AllocationRequest_AcceptV2Result => None - case v: TSAR_AllocationInstructionResultV2 => - v.allocationInstructionResultValue.output match { - case completed: AllocationInstructionResult_Completed => - Some(completed.allocationCid) - case other => - fail(s"Expected AllocationInstructionResult_Completed but got $other") - } - case other => - fail(s"Expected TSAR_AllocationResultV2 but got $other") - } - .collect { case Some(cid) => cid } - - (aliceAllocationCids, aliceAllocateTx) - } - } - - val (bobAllocationCids, bobAllocateTx) = clue( - "Bob uses the BatchingUtilityV2 to accept the request and create two allocations in a single tx" - ) { - // Same UpdateExternalPartyConfigStateTrigger/LOCAL_VERDICT_INACTIVE_CONTRACTS logic - // as with alice's usage of BatchingUtilityV2 above. - eventuallySucceeds() { - val batchingUtility = batchingUtilityIds(bobParty) - val amuletSpec = bobAllocationRequest.contract.payload.allocations.asScala - .filter(_.admin == dsoParty.toProtoPrimitive) - .loneElement - val usdcSpec = bobAllocationRequest.contract.payload.allocations.asScala - .filter(_.admin == ttAdminParty.toProtoPrimitive) - .loneElement - val amuletAllocationFactory = sv1ScanBackend.getAllocationFactoryV2( - new allocationinstructionv2.AllocationFactory_Allocate( - bobAllocationRequest.contract.payload.settlement, - amuletSpec, - bobAllocationRequest.contract.payload.requestedAt, - java.util.List.of(), // bob has no amulets - emptyExtraArgs, - java.util.List.of(bobParty.toProtoPrimitive), + val (aliceAllocationCids, aliceAllocateTx) = clue( + "Alice uses the BatchingUtilityV2 to create two allocations and accept the allocation request in a single tx" + ) { + // UpdateExternalPartyConfigStateTrigger might run concurrently and cause a LOCAL_VERDICT_INACTIVE_CONTRACTS + // because of the ExternalPartyConfigState being updated. + // In the real world, we expect the venue to also just retry re-fetching all contexts + eventuallySucceeds() { + val batchingUtility = batchingUtilityIds(aliceParty) + val aliceAmulets = aliceWalletClient + .list() + .amulets + .map(_.contract.contractId.toInterface(holdingv2.Holding.INTERFACE)) + val amuletSpec = aliceAllocationRequest.contract.payload.allocations.asScala + .filter(_.admin == dsoParty.toProtoPrimitive) + .loneElement + val usdcSpec = aliceAllocationRequest.contract.payload.allocations.asScala + .filter(_.admin == ttAdminParty.toProtoPrimitive) + .loneElement + val amuletAllocationFactory = sv1ScanBackend.getAllocationFactoryV2( + new allocationinstructionv2.AllocationFactory_Allocate( + aliceAllocationRequest.contract.payload.settlement, + amuletSpec, + aliceAllocationRequest.contract.payload.requestedAt, + aliceAmulets.asJava, + emptyExtraArgs, + java.util.List.of(aliceParty.toProtoPrimitive), + ) ) - ) - val bobUsdcHoldings = getHoldings(bobParty, bobValidatorBackend) - .map(_.contractId) - .map(id => new holdingv2.Holding.ContractId(id)) - val usdcContext = registry.getContext( - bobUsdcHoldings - ) - val bobAllocateUpdate = batchingUtility - .exerciseBatchingUtility_ExecuteBatch( - new HoldingMap( - Map( - new ScopedAccount( - dsoParty.toProtoPrimitive, - basicAccount(bobParty), - ) -> Map[String, java.util.List[holdingv2.Holding.ContractId]]().asJava, - new ScopedAccount( - ttAdminParty.toProtoPrimitive, - basicAccount(bobParty), - ) -> Map[String, java.util.List[holdingv2.Holding.ContractId]]( - usdcInstrumentName -> bobUsdcHoldings.asJava - ).asJava, // alice has no USDC here yet - ).asJava - ), - java.util.List.of( - new TSA_AllocationFactory_AllocateV2( - new ChoiceCall[AllocationFactory_Allocate]( - new metadatav1.AnyContract.ContractId( - amuletAllocationFactory.factoryId.contractId - ), - amuletAllocationFactory.args, - ) - ), - new TSA_AllocationFactory_AllocateV2( - new ChoiceCall[AllocationFactory_Allocate]( - new metadatav1.AnyContract.ContractId(tokenRulesId.contractId), - new allocationinstructionv2.AllocationFactory_Allocate( - bobAllocationRequest.contract.payload.settlement, - usdcSpec, - bobAllocationRequest.contract.payload.requestedAt, - java.util.List.of(), - new metadatav1.ExtraArgs(usdcContext.choiceContext, emptyMetadata), - java.util.List.of(bobParty.toProtoPrimitive), - ), - ) + val usdcContext = registry.getContext(Seq.empty) + val aliceAllocateUpdate = batchingUtility + .exerciseBatchingUtility_ExecuteBatch( + new HoldingMap( + Map( + new ScopedAccount( + dsoParty.toProtoPrimitive, + basicAccount(aliceParty), + ) -> Map[String, java.util.List[holdingv2.Holding.ContractId]]( + amuletInstrumentIdName -> aliceAmulets.asJava + ).asJava, + new ScopedAccount( + ttAdminParty.toProtoPrimitive, + basicAccount(aliceParty), + ) -> Map + .empty[String, java.util.List[holdingv2.Holding.ContractId]] + .asJava, // alice has no USDC here yet + ).asJava ), - new TSA_AllocationRequest_AcceptV2( - new ChoiceCall[AllocationRequest_Accept]( - new metadatav1.AnyContract.ContractId( - bobAllocationRequest.contract.contractId.contractId - ), - new AllocationRequest_Accept( - java.util.List.of(bobParty.toProtoPrimitive), - new metadatav1.ExtraArgs(usdcContext.choiceContext, emptyMetadata), - ), - ) + java.util.List.of( + new TSA_AllocationFactory_AllocateV2( + new ChoiceCall[AllocationFactory_Allocate]( + new metadatav1.AnyContract.ContractId( + amuletAllocationFactory.factoryId.contractId + ), + amuletAllocationFactory.args, + ) + ), + new TSA_AllocationFactory_AllocateV2( + new ChoiceCall[AllocationFactory_Allocate]( + new metadatav1.AnyContract.ContractId(tokenRulesId.contractId), + new allocationinstructionv2.AllocationFactory_Allocate( + aliceAllocationRequest.contract.payload.settlement, + usdcSpec, + aliceAllocationRequest.contract.payload.requestedAt, + java.util.List.of(), + new metadatav1.ExtraArgs(usdcContext.choiceContext, emptyMetadata), + java.util.List.of(aliceParty.toProtoPrimitive), + ), + ) + ), + new TSA_AllocationRequest_AcceptV2( + new ChoiceCall[AllocationRequest_Accept]( + new metadatav1.AnyContract.ContractId( + aliceAllocationRequest.contract.contractId.contractId + ), + new AllocationRequest_Accept( + java.util.List.of(aliceParty.toProtoPrimitive), + amuletAllocationFactory.args.extraArgs, + ), + ) + ), ), - ), - true, - ) - val bobAllocateTx = - bobValidatorBackend.participantClientWithAdminToken.ledger_api_extensions.commands - .submitJava( - userId = bobValidatorBackend.config.ledgerApiUser, - actAs = Seq(bobParty), - readAs = Seq(bobParty), - commands = bobAllocateUpdate.commands().asScala.toSeq, - disclosedContracts = - amuletAllocationFactory.disclosedContracts ++ usdcContext.disclosedContracts, + true, ) + val aliceAllocateTx = + aliceValidatorBackend.participantClientWithAdminToken.ledger_api_extensions.commands + .submitJava( + userId = aliceValidatorBackend.config.ledgerApiUser, + actAs = Seq(aliceParty), + readAs = Seq(aliceParty), + commands = aliceAllocateUpdate.commands().asScala.toSeq, + disclosedContracts = + amuletAllocationFactory.disclosedContracts ++ usdcContext.disclosedContracts, + ) - val bobAllocationCids = SpliceLedgerConnection - .decodeExerciseResult( - bobAllocateUpdate, - bobAllocateTx, - ) - .exerciseResult - .actionResults - .asScala - .map { - case _: TSAR_AllocationRequest_AcceptV2Result => None - case v: TSAR_AllocationInstructionResultV2 => - v.allocationInstructionResultValue.output match { - case completed: AllocationInstructionResult_Completed => - Some(completed.allocationCid) - case other => - fail(s"Expected AllocationInstructionResult_Completed but got $other") - } - case other => - fail(s"Expected TSAR_AllocationResultV2 but got $other") - } - .collect { case Some(cid) => cid } - - (bobAllocationCids, bobAllocateTx) + val aliceAllocationCids = SpliceLedgerConnection + .decodeExerciseResult( + aliceAllocateUpdate, + aliceAllocateTx, + ) + .exerciseResult + .actionResults + .asScala + .map { + case _: TSAR_AllocationRequest_AcceptV2Result => None + case v: TSAR_AllocationInstructionResultV2 => + v.allocationInstructionResultValue.output match { + case completed: AllocationInstructionResult_Completed => + Some(completed.allocationCid) + case other => + fail(s"Expected AllocationInstructionResult_Completed but got $other") + } + case other => + fail(s"Expected TSAR_AllocationResultV2 but got $other") + } + .collect { case Some(cid) => cid } + + (aliceAllocationCids, aliceAllocateTx) + } } - } - val (settleTradeTx, _) = actAndCheck( - "Venue settles the trade", { + val (bobAllocationCids, bobAllocateTx) = clue( + "Bob uses the BatchingUtilityV2 to accept the request and create two allocations in a single tx" + ) { // Same UpdateExternalPartyConfigStateTrigger/LOCAL_VERDICT_INACTIVE_CONTRACTS logic // as with alice's usage of BatchingUtilityV2 above. eventuallySucceeds() { - val allAllocations = { - venueValidator.participantClientWithAdminToken.ledger_api.state.acs.of_party( - party = venueParty, - filterInterfaces = Seq(allocationv2.Allocation.TEMPLATE_ID).map(templateId => - TemplateId( - templateId.getPackageId, - templateId.getModuleName, - templateId.getEntityName, - ) - ), - includeCreatedEventBlob = true, + val batchingUtility = batchingUtilityIds(bobParty) + val amuletSpec = bobAllocationRequest.contract.payload.allocations.asScala + .filter(_.admin == dsoParty.toProtoPrimitive) + .loneElement + val usdcSpec = bobAllocationRequest.contract.payload.allocations.asScala + .filter(_.admin == ttAdminParty.toProtoPrimitive) + .loneElement + val amuletAllocationFactory = sv1ScanBackend.getAllocationFactoryV2( + new allocationinstructionv2.AllocationFactory_Allocate( + bobAllocationRequest.contract.payload.settlement, + amuletSpec, + bobAllocationRequest.contract.payload.requestedAt, + java.util.List.of(), // bob has no amulets + emptyExtraArgs, + java.util.List.of(bobParty.toProtoPrimitive), ) - } - // sanity check - (bobAllocationCids ++ aliceAllocationCids).foreach { cid => - allAllocations - .find(_.contractId == cid.contractId) - .valueOrFail(s"No allocation found for cid $cid") - } - val amuletAllocations = - allAllocations.filter(_.event.signatories.contains(dsoParty.toProtoPrimitive)) - val usdAllocations = - allAllocations.filter(_.event.signatories.contains(ttAdminParty.toProtoPrimitive)) - val settleBatch = new allocationv2.SettlementFactory_SettleBatch( - new allocationv2.SettlementInfo( - java.util.List.of(venueParty.toProtoPrimitive), - "OTCTrade", - java.util.Optional - .of(new metadatav1.AnyContract.ContractId(otcTrade.id.contractId)), - emptyMetadata, - ), - transferLegsFromTrade(otcTrade).asJava, - allAllocations - .map(alloc => - new allocationv2.FinalizedAllocation( - new allocationv2.Allocation.ContractId(alloc.contractId), - java.util.List.of(), - java.util.Optional.empty[java.util.Map[String, java.math.BigDecimal]](), - ) - ) - .asJava, - /*actors = */ java.util.List.of(venueParty.toProtoPrimitive), - emptyExtraArgs, ) - val amuletContext = sv1ScanBackend.getSettlementFactoryV2(settleBatch) val bobUsdcHoldings = getHoldings(bobParty, bobValidatorBackend) .map(_.contractId) .map(id => new holdingv2.Holding.ContractId(id)) val usdcContext = registry.getContext( bobUsdcHoldings ) - venueValidator.participantClientWithAdminToken.ledger_api_extensions.commands - .submitJava( - actAs = Seq(venueParty), - commands = otcTrade.id - .exerciseOTCTrade_Settle( - Map[String, tradingappv2.SettlementBatch]( - dsoParty.toProtoPrimitive -> new SettlementBatchV2( - amuletAllocations - .map(alloc => new allocationv2.Allocation.ContractId(alloc.contractId)) - .asJava, + val bobAllocateUpdate = batchingUtility + .exerciseBatchingUtility_ExecuteBatch( + new HoldingMap( + Map( + new ScopedAccount( + dsoParty.toProtoPrimitive, + basicAccount(bobParty), + ) -> Map[String, java.util.List[holdingv2.Holding.ContractId]]().asJava, + new ScopedAccount( + ttAdminParty.toProtoPrimitive, + basicAccount(bobParty), + ) -> Map[String, java.util.List[holdingv2.Holding.ContractId]]( + usdcInstrumentName -> bobUsdcHoldings.asJava + ).asJava, // alice has no USDC here yet + ).asJava + ), + java.util.List.of( + new TSA_AllocationFactory_AllocateV2( + new ChoiceCall[AllocationFactory_Allocate]( + new metadatav1.AnyContract.ContractId( + amuletAllocationFactory.factoryId.contractId + ), + amuletAllocationFactory.args, + ) + ), + new TSA_AllocationFactory_AllocateV2( + new ChoiceCall[AllocationFactory_Allocate]( + new metadatav1.AnyContract.ContractId(tokenRulesId.contractId), + new allocationinstructionv2.AllocationFactory_Allocate( + bobAllocationRequest.contract.payload.settlement, + usdcSpec, + bobAllocationRequest.contract.payload.requestedAt, java.util.List.of(), - amuletContext.factoryId, - amuletContext.args.extraArgs, + new metadatav1.ExtraArgs(usdcContext.choiceContext, emptyMetadata), + java.util.List.of(bobParty.toProtoPrimitive), ), - ttAdminParty.toProtoPrimitive -> new SettlementBatchV2( - usdAllocations - .map(alloc => new allocationv2.Allocation.ContractId(alloc.contractId)) - .asJava, - java.util.List.of( - new tradingappv2.MissingAllocation( - java.util.Optional.empty(), - tokenRulesId.toInterface( - allocationinstructionv2.AllocationFactory.INTERFACE - ), - new allocationinstructionv2.AllocationFactory_Allocate( - new allocationv2.SettlementInfo( - java.util.List.of(venueParty.toProtoPrimitive), - "OTCTradeProposal", - java.util.Optional.of( - new metadatav1.AnyContract.ContractId(otcTrade.id.contractId) - ), - emptyMetadata, - ), - new allocationv2.AllocationSpecification( - ttAdminParty.toProtoPrimitive, - basicAccount(venueParty), - java.util.List.of( - new allocationv2.TransferLegSide( - "alicetovenue0.2USDC", - allocationv2.TransferSide.RECEIVERSIDE, - basicAccount(aliceParty), - BigDecimal(0.2).bigDecimal, - usdcInstrumentName, - emptyMetadata, - ) - ), - java.util.Optional.empty(), - java.util.Optional.empty(), - false, - emptyMetadata, - ), - Instant.now(), - java.util.List.of(), - new metadatav1.ExtraArgs(usdcContext.choiceContext, emptyMetadata), - java.util.List.of(venueParty.toProtoPrimitive), - ), - ) - ), - new allocationv2.SettlementFactory.ContractId(tokenRulesId.contractId), + ) + ), + new TSA_AllocationRequest_AcceptV2( + new ChoiceCall[AllocationRequest_Accept]( + new metadatav1.AnyContract.ContractId( + bobAllocationRequest.contract.contractId.contractId + ), + new AllocationRequest_Accept( + java.util.List.of(bobParty.toProtoPrimitive), new metadatav1.ExtraArgs(usdcContext.choiceContext, emptyMetadata), ), - ).asJava, - java.util.List.of(), - ) - .commands() - .asScala - .toSeq, - disclosedContracts = - usdcContext.disclosedContracts ++ amuletContext.disclosedContracts, + ) + ), + ), + true, + ) + val bobAllocateTx = + bobValidatorBackend.participantClientWithAdminToken.ledger_api_extensions.commands + .submitJava( + userId = bobValidatorBackend.config.ledgerApiUser, + actAs = Seq(bobParty), + readAs = Seq(bobParty), + commands = bobAllocateUpdate.commands().asScala.toSeq, + disclosedContracts = + amuletAllocationFactory.disclosedContracts ++ usdcContext.disclosedContracts, + ) + + val bobAllocationCids = SpliceLedgerConnection + .decodeExerciseResult( + bobAllocateUpdate, + bobAllocateTx, ) + .exerciseResult + .actionResults + .asScala + .map { + case _: TSAR_AllocationRequest_AcceptV2Result => None + case v: TSAR_AllocationInstructionResultV2 => + v.allocationInstructionResultValue.output match { + case completed: AllocationInstructionResult_Completed => + Some(completed.allocationCid) + case other => + fail(s"Expected AllocationInstructionResult_Completed but got $other") + } + case other => + fail(s"Expected TSAR_AllocationResultV2 but got $other") + } + .collect { case Some(cid) => cid } + + (bobAllocationCids, bobAllocateTx) } - }, - )( - "The balances are updated", - _ => { - aliceWalletClient.balance().unlockedQty should be(aliceCCBalanceBefore - 100) - bobWalletClient.balance().unlockedQty should be(100) - - getUsdcBalance(bobParty, bobValidatorBackend) should be(bobOfferMintAmount - 15) - getUsdcBalance(aliceParty, aliceValidatorBackend) should be(15 - 0.2) - getUsdcBalance(venueParty, venueValidator) should be(0.2) - }, - ) + } - val events = Seq( - createTradeTx -> "Create Trade", - createAllocationRequestsTx -> "Create Allocation Requests", - aliceAllocateTx -> "Alice Allocations", - bobAllocateTx -> "Bob Allocations", - settleTradeTx -> "Settle Trade", - ).map { case (tx, name) => - val updateId = tx.getUpdateId - name -> clue(s"Checking traffic & activity records for '$name'") { - eventually() { - inside(sv1ScanBackend.getEventById(updateId, None)) { - case Some( - item @ EventHistoryItem( - _, - Some(_), - Some(_), - Some(_), + val (settleTradeTx, _) = actAndCheck( + "Venue settles the trade", { + // Same UpdateExternalPartyConfigStateTrigger/LOCAL_VERDICT_INACTIVE_CONTRACTS logic + // as with alice's usage of BatchingUtilityV2 above. + eventuallySucceeds() { + val allAllocations = { + venueValidator.participantClientWithAdminToken.ledger_api.state.acs.of_party( + party = venueParty, + filterInterfaces = Seq(allocationv2.Allocation.TEMPLATE_ID).map(templateId => + TemplateId( + templateId.getPackageId, + templateId.getModuleName, + templateId.getEntityName, + ) + ), + includeCreatedEventBlob = true, + ) + } + // sanity check + (bobAllocationCids ++ aliceAllocationCids).foreach { cid => + allAllocations + .find(_.contractId == cid.contractId) + .valueOrFail(s"No allocation found for cid $cid") + } + val amuletAllocations = + allAllocations.filter(_.event.signatories.contains(dsoParty.toProtoPrimitive)) + val usdAllocations = + allAllocations.filter(_.event.signatories.contains(ttAdminParty.toProtoPrimitive)) + val settleBatch = new allocationv2.SettlementFactory_SettleBatch( + new allocationv2.SettlementInfo( + java.util.List.of(venueParty.toProtoPrimitive), + "OTCTrade", + java.util.Optional + .of(new metadatav1.AnyContract.ContractId(otcTrade.id.contractId)), + emptyMetadata, + ), + transferLegsFromTrade(otcTrade).asJava, + allAllocations + .map(alloc => + new allocationv2.FinalizedAllocation( + new allocationv2.Allocation.ContractId(alloc.contractId), + java.util.List.of(), + java.util.Optional.empty[java.util.Map[String, java.math.BigDecimal]](), ) - ) => - EventHistoryItem.encodeEventHistoryItem(item) + ) + .asJava, + /*actors = */ java.util.List.of(venueParty.toProtoPrimitive), + emptyExtraArgs, + ) + val amuletContext = sv1ScanBackend.getSettlementFactoryV2(settleBatch) + val bobUsdcHoldings = getHoldings(bobParty, bobValidatorBackend) + .map(_.contractId) + .map(id => new holdingv2.Holding.ContractId(id)) + val usdcContext = registry.getContext( + bobUsdcHoldings + ) + venueValidator.participantClientWithAdminToken.ledger_api_extensions.commands + .submitJava( + actAs = Seq(venueParty), + commands = otcTrade.id + .exerciseOTCTrade_Settle( + Map[String, tradingappv2.SettlementBatch]( + dsoParty.toProtoPrimitive -> new SettlementBatchV2( + amuletAllocations + .map(alloc => new allocationv2.Allocation.ContractId(alloc.contractId)) + .asJava, + java.util.List.of(), + amuletContext.factoryId, + amuletContext.args.extraArgs, + ), + ttAdminParty.toProtoPrimitive -> new SettlementBatchV2( + usdAllocations + .map(alloc => new allocationv2.Allocation.ContractId(alloc.contractId)) + .asJava, + java.util.List.of( + new tradingappv2.MissingAllocation( + java.util.Optional.empty(), + tokenRulesId.toInterface( + allocationinstructionv2.AllocationFactory.INTERFACE + ), + new allocationinstructionv2.AllocationFactory_Allocate( + new allocationv2.SettlementInfo( + java.util.List.of(venueParty.toProtoPrimitive), + "OTCTradeProposal", + java.util.Optional.of( + new metadatav1.AnyContract.ContractId(otcTrade.id.contractId) + ), + emptyMetadata, + ), + new allocationv2.AllocationSpecification( + ttAdminParty.toProtoPrimitive, + basicAccount(venueParty), + java.util.List.of( + new allocationv2.TransferLegSide( + "alicetovenue0.2USDC", + allocationv2.TransferSide.RECEIVERSIDE, + basicAccount(aliceParty), + BigDecimal(0.2).bigDecimal, + usdcInstrumentName, + emptyMetadata, + ) + ), + java.util.Optional.empty(), + java.util.Optional.empty(), + false, + emptyMetadata, + ), + Instant.now(), + java.util.List.of(), + new metadatav1.ExtraArgs(usdcContext.choiceContext, emptyMetadata), + java.util.List.of(venueParty.toProtoPrimitive), + ), + ) + ), + new allocationv2.SettlementFactory.ContractId(tokenRulesId.contractId), + new metadatav1.ExtraArgs(usdcContext.choiceContext, emptyMetadata), + ), + ).asJava, + java.util.List.of(), + ) + .commands() + .asScala + .toSeq, + disclosedContracts = + usdcContext.disclosedContracts ++ amuletContext.disclosedContracts, + ) + } + }, + )( + "The balances are updated", + _ => { + aliceWalletClient.balance().unlockedQty should be(aliceCCBalanceBefore - 100) + bobWalletClient.balance().unlockedQty should be(100) + + getUsdcBalance(bobParty, bobValidatorBackend) should be(bobOfferMintAmount - 15) + getUsdcBalance(aliceParty, aliceValidatorBackend) should be(15 - 0.2) + getUsdcBalance(venueParty, venueValidator) should be(0.2) + }, + ) + + val events = Seq( + createTradeTx -> "Create Trade", + createAllocationRequestsTx -> "Create Allocation Requests", + aliceAllocateTx -> "Alice Allocations", + bobAllocateTx -> "Bob Allocations", + settleTradeTx -> "Settle Trade", + ).map { case (tx, name) => + val updateId = tx.getUpdateId + name -> clue(s"Checking traffic & activity records for '$name'") { + eventually() { + inside(sv1ScanBackend.getEventById(updateId, None)) { + case Some( + item @ EventHistoryItem( + _, + Some(_), + Some(_), + Some(_), + ) + ) => + EventHistoryItem.encodeEventHistoryItem(item) + } } } } - } - val json = io.circe.JsonObject(events*) - val savePath = java.io.File.createTempFile("test_token_v2_settlement_results", ".json").toPath - Files.writeString(savePath, json.toJson.spaces2) + val json = io.circe.JsonObject(events*) + val savePath = + java.io.File.createTempFile("test_token_v2_settlement_results", ".json").toPath + Files.writeString(savePath, json.toJson.spaces2) - logger.info(s"Traffic & Activity Records results written to $savePath") + logger.info(s"Traffic & Activity Records results written to $savePath") + } } } diff --git a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/TrafficBasedRewardsSvAppTimeBasedIntegrationTest.scala b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/TrafficBasedRewardsSvAppTimeBasedIntegrationTest.scala index 6872258e32..13a38c0fd0 100644 --- a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/TrafficBasedRewardsSvAppTimeBasedIntegrationTest.scala +++ b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/TrafficBasedRewardsSvAppTimeBasedIntegrationTest.scala @@ -36,11 +36,13 @@ import org.lfdecentralizedtrust.splice.sv.automation.RewardMetricsTrigger import org.lfdecentralizedtrust.splice.sv.automation.confirmation.{ CalculateRewardsDryRunTrigger, CalculateRewardsTrigger, + SummarizingMiningRoundTrigger, } import org.lfdecentralizedtrust.splice.sv.automation.delegatebased.{ ProcessRewardsDryRunTrigger, ProcessRewardsTrigger, } +import org.lfdecentralizedtrust.splice.scan.admin.api.client.BftScanConnection import org.lfdecentralizedtrust.splice.scan.automation.RewardComputationTrigger import org.lfdecentralizedtrust.splice.sv.config.InitialRewardConfig import org.lfdecentralizedtrust.splice.util.{ @@ -352,6 +354,45 @@ class TrafficBasedRewardsSvAppTimeBasedIntegrationTest confirmMismatchingRootHashIsFlagged(bobParty) } + // sv2's CalculateRewardsTrigger and SummarizingMiningRoundTrigger report the + // scan URIs that formed the BFT consensus at INFO. This method captures the + // logs emitted while running the 'body' argument and asserts that sv2 + // obtained both the root-hash and the reward accounting totals for 'round' + // via BFT read from sv1 and sv4. + private def withExpectedRewardTriggersLogging[A](round: Long)( + body: => A + ): A = { + val bftReadLogs = + (SuppressionRule.forLogger[CalculateRewardsTrigger] || + SuppressionRule.forLogger[SummarizingMiningRoundTrigger]) && + SuppressionRule.LevelAndAbove(Level.INFO) + + loggerFactory.assertEventuallyLogsSeq(bftReadLogs)( + body, + logs => { + // sv3 is stopped and sv2's own scan is not part of its peer BFT connection, + // so only sv1's and sv4's scans can form the consensus. + val expectedScanUris = Set("http://localhost:5012", "http://localhost:5312") + def bftReadLogged(subject: String) = + forAtLeast(1, logs) { log => + val prefix = + s"Obtained the $subject for round $round via BFT read from scans: " + log.loggerName should include("SV=sv2") + log.message should include(prefix) + val scanUris = log.message + .substring(log.message.indexOf(prefix) + prefix.length) + .stripSuffix(".") + .split(", ") + .toSeq + scanUris.size should be(1) + forAll(scanUris)(uri => expectedScanUris should contain(uri)) + } + bftReadLogged("root-hash") + bftReadLogged("reward accounting totals") + }, + ) + } + private def metricValue( node: LocalInstanceReference, name: String, @@ -385,99 +426,101 @@ class TrafficBasedRewardsSvAppTimeBasedIntegrationTest // Pausing this ensures that the root-hash is not calculated while we advance round val sv2RewardComputation = sv2ScanBackend.automation.trigger[RewardComputationTrigger] - // Here we ensure that SV2 has done ingestion of app-activity for the round just closed - // But then its AppActivityRecordMetaT is bumped so that it cannot compute the - // root-hash for the round. - val (calculateRewardsCid, round) = setTriggersWithin( - triggersToPauseAtStart = Seq(sv2CalculateRewards, sv2RewardComputation) - ) { - val round = oldestOpenRound - doTransfer(bobParty) - // Note: we can't use advanceRoundsToNextRoundOpening here, as it blocks - // on summarizing and issuing round to complete, and here the - // summarizing round will block until the sv2 provides the round totals - // via bft read. - advanceTimeAndWaitForRoundOpening - - val (calculateRewardsCid, rootHash) = - clue( - s"Round $round just closed: its CalculateRewardsV2 exists and sv1 serves root-hash" - ) { - eventually() { - val calc = sv1Backend.appState.dsoStore - .listCalculateRewardsV2() - .futureValue - .filterNot(_.payload.dryRun) - .find(_.payload.round.number == round) - .value - val rootHash = inside(sv1ScanBackend.getRewardAccountingRootHash(round)) { - case GetRewardAccountingRootHashResponse.members.RewardAccountingRootHashOk(h) => - h.rootHash + val round = oldestOpenRound + withExpectedRewardTriggersLogging(round) { + // Here we ensure that SV2 has done ingestion of app-activity for the round just closed + // But then its AppActivityRecordMetaT is bumped so that it cannot compute the + // root-hash for the round. + val calculateRewardsCid = setTriggersWithin( + triggersToPauseAtStart = Seq(sv2CalculateRewards, sv2RewardComputation) + ) { + doTransfer(bobParty) + // Note: we can't use advanceRoundsToNextRoundOpening here, as it blocks + // on summarizing and issuing round to complete, and here the + // summarizing round will block until the sv2 provides the round totals + // via bft read. + advanceTimeAndWaitForRoundOpening + + val (calculateRewardsCid, rootHash) = + clue( + s"Round $round just closed: its CalculateRewardsV2 exists and sv1 serves root-hash" + ) { + eventually() { + val calc = sv1Backend.appState.dsoStore + .listCalculateRewardsV2() + .futureValue + .filterNot(_.payload.dryRun) + .find(_.payload.round.number == round) + .value + val rootHash = inside(sv1ScanBackend.getRewardAccountingRootHash(round)) { + case GetRewardAccountingRootHashResponse.members.RewardAccountingRootHashOk(h) => + h.rootHash + } + (calc.contractId, rootHash) } - (calc.contractId, rootHash) } - } - clue(s"Only sv1 and sv4 confirm round $round, so it is not yet processed") { - eventually() { - val startProcessingAction = new ARC_AmuletRules( - new CRARC_StartProcessingRewardsV2( - new AmuletRules_StartProcessingRewardsV2(calculateRewardsCid, new Hash(rootHash)) + clue(s"Only sv1 and sv4 confirm round $round, so it is not yet processed") { + eventually() { + val startProcessingAction = new ARC_AmuletRules( + new CRARC_StartProcessingRewardsV2( + new AmuletRules_StartProcessingRewardsV2(calculateRewardsCid, new Hash(rootHash)) + ) ) - ) + sv1Backend.appState.dsoStore + .listConfirmations(startProcessingAction) + .futureValue should have size 2 + } sv1Backend.appState.dsoStore - .listConfirmations(startProcessingAction) - .futureValue should have size 2 + .listOldestSummarizingMiningRounds() + .futureValue + .map(_.payload.round.number) should contain(round) } - sv1Backend.appState.dsoStore - .listOldestSummarizingMiningRounds() - .futureValue - .map(_.payload.round.number) should contain(round) - } - // This is trying to simulate AppActivityRecordMetaT's userVersion bump - // albeit in a direct way, to avoid restart of scan app, etc. - actAndCheck( - s"Reset sv2's earliest-ingested round to $round", { - val sv2Db = sv2ScanBackend.appState.storage match { - case db: DbStorage => db - case other => fail(s"Expected DbStorage") - } - implicit val closeContext: CloseContext = CloseContext(sv2Db) - sv2Db - .update_( - sqlu"""update app_activity_record_meta - set earliest_ingested_round = $round, - last_archived_round = null""", - "test.increaseAppActivityMeta_EarliestIngestedRound", - ) - .futureValueUS - }, - )( - s"sv2's own scan now answers CannotProvide for round $round", - _ => - sv2ScanBackend.getRewardAccountingRootHash(round) shouldBe - a[GetRewardAccountingRootHashResponse.members.RewardAccountingRootHashCannotProvide], - ) + // This is trying to simulate AppActivityRecordMetaT's userVersion bump + // albeit in a direct way, to avoid restart of scan app, etc. + actAndCheck( + s"Reset sv2's earliest-ingested round to $round", { + val sv2Db = sv2ScanBackend.appState.storage match { + case db: DbStorage => db + case other => fail(s"Expected DbStorage") + } + implicit val closeContext: CloseContext = CloseContext(sv2Db) + sv2Db + .update_( + sqlu"""update app_activity_record_meta + set earliest_ingested_round = $round, + last_archived_round = null""", + "test.increaseAppActivityMeta_EarliestIngestedRound", + ) + .futureValueUS + }, + )( + s"sv2's own scan now answers CannotProvide for round $round", + _ => + sv2ScanBackend.getRewardAccountingRootHash(round) shouldBe + a[GetRewardAccountingRootHashResponse.members.RewardAccountingRootHashCannotProvide], + ) - (calculateRewardsCid, round) - } + calculateRewardsCid + } - // setTriggersWithin has resumed sv2's CalculateRewardsTrigger. sv3 is stopped and sv2's own - // scan CannotProvide, so the deciding 3rd confirmation can only come from sv2 via bft read. - clue(s"sv2's own scan still answers CannotProvide for round $round") { - sv2ScanBackend.getRewardAccountingRootHash(round) shouldBe - a[GetRewardAccountingRootHashResponse.members.RewardAccountingRootHashCannotProvide] - } + // setTriggersWithin has resumed sv2's CalculateRewardsTrigger. sv3 is stopped and sv2's own + // scan CannotProvide, so the deciding 3rd confirmation can only come from sv2 via bft read. + clue(s"sv2's own scan still answers CannotProvide for round $round") { + sv2ScanBackend.getRewardAccountingRootHash(round) shouldBe + a[GetRewardAccountingRootHashResponse.members.RewardAccountingRootHashCannotProvide] + } - clue( - s"sv2 reads round $round from the sv1 and sv4, and supplies the 3rd confirmation vote" - ) { - eventually() { - sv1Backend.appState.dsoStore - .listCalculateRewardsV2() - .futureValue - .map(_.contractId) should not contain calculateRewardsCid + clue( + s"sv2 reads round $round from the sv1 and sv4, and supplies the 3rd confirmation vote" + ) { + eventually() { + sv1Backend.appState.dsoStore + .listCalculateRewardsV2() + .futureValue + .map(_.contractId) should not contain calculateRewardsCid + } } } @@ -515,15 +558,38 @@ class TrafficBasedRewardsSvAppTimeBasedIntegrationTest } } finally { otherProcessRewardsTriggers.foreach(_.resume()) - clue("Restart sv3") { - sv3ScanBackend.start() - sv3Backend.start() - sv3Backend.waitForInitialization( - timeout = NonNegativeDuration.tryFromDuration(120.seconds) - ) - sv3ScanBackend.waitForInitialization( - timeout = NonNegativeDuration.tryFromDuration(120.seconds) - ) + // On restart, sv3 catches up on the round that was processed while sv3 + // was down. The reward triggers may fire before that round + // advances. sv2's own scan still answers 'CannotProvide' for that round + // (its earliest-ingested round was bumped above), so it contributes an + // 'IgnoreResponse' to sv3's BFT reads, which 'BftScanConnection' logs at + // WARN as "The following Scan URLs disagreed with consensus". These WARNs + // are an expected consequence of the 'CannotProvide' scenario under test, + // so we suppress them (targeted to 'BftScanConnection' WARNs) to keep the + // `sbt checkErrors` log-scan gate green. + // + // The same supression happens in 'withExpectedRewardTriggersLogging' but + // + // 1. 'withExpectedRewardTriggersLogging' targets a narrow part of the try + // block and doesn't expand into this finally block, and + // 2. 'withExpectedRewardTriggersLogging' has strict expectation about the + // logs when rewards trigger fire. Here triggers may or may not fire + // -- it's a race between sv3 catching up and triggers firing. We + // can't guarantee that triggers fire => can't expect that WARNs will + // appear. So we just supress them instead of expecting them. + loggerFactory.suppress( + SuppressionRule.forLogger[BftScanConnection] && SuppressionRule.Level(Level.WARN) + ) { + clue("Restart sv3") { + sv3ScanBackend.start() + sv3Backend.start() + sv3Backend.waitForInitialization( + timeout = NonNegativeDuration.tryFromDuration(120.seconds) + ) + sv3ScanBackend.waitForInitialization( + timeout = NonNegativeDuration.tryFromDuration(120.seconds) + ) + } } } } diff --git a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/WalletIntegrationTest.scala b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/WalletIntegrationTest.scala index 3cc5e4bfc3..1c67b378f1 100644 --- a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/WalletIntegrationTest.scala +++ b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/WalletIntegrationTest.scala @@ -2,7 +2,6 @@ package org.lfdecentralizedtrust.splice.integration.tests import org.lfdecentralizedtrust.splice.auth.AuthUtil import org.lfdecentralizedtrust.splice.codegen.java.splice.amulet as amuletCodegen -import org.lfdecentralizedtrust.splice.codegen.java.splice.types.Round import org.lfdecentralizedtrust.splice.codegen.java.splice.wallet.payment as walletCodegen import org.lfdecentralizedtrust.splice.codegen.java.splice.wallet.transferpreapproval.TransferPreapprovalProposal import org.lfdecentralizedtrust.splice.http.v0.definitions.TapRequest @@ -12,11 +11,7 @@ import org.lfdecentralizedtrust.splice.integration.EnvironmentDefinition import org.lfdecentralizedtrust.splice.integration.tests.SpliceTests.BracketSynchronous.bracket import org.lfdecentralizedtrust.splice.integration.tests.WalletTxLogTestUtil import org.lfdecentralizedtrust.splice.store.MultiDomainAcsStore.ContractState -import org.lfdecentralizedtrust.splice.util.{ - SpliceUtil, - WalletTestUtil, - JavaDecodeUtil as DecodeUtil, -} +import org.lfdecentralizedtrust.splice.util.{WalletTestUtil, JavaDecodeUtil as DecodeUtil} import org.lfdecentralizedtrust.splice.validator.automation.AcceptTransferPreapprovalProposalTrigger import org.lfdecentralizedtrust.splice.wallet.admin.api.client.commands.HttpWalletAppClient.CreateTransferPreapprovalResponse import org.lfdecentralizedtrust.splice.wallet.store.{ @@ -60,31 +55,29 @@ class WalletIntegrationTest "A wallet" should { - // TODO (#2336): unignore this test - "tap stupid amount" ignore { implicit env => + val tapLimit = 100000000 + + s"tap $tapLimit amount" in { implicit env => import com.digitalasset.daml.lf.data.Numeric val aliceParty = onboardWalletUser(aliceWalletClient, aliceValidatorBackend) val round = sv1ScanBackend.getLatestOpenMiningRound(env.environment.clock.now) val price = round.contract.payload.amuletPrice val decimalScale = Numeric.Scale.assertFromInt(10) - // We subtract one to allow some slack in back/forth conversions from CC to USD. Otherwise, - // the command gets rejected by the participant and we test nothing. - val maxDecimal = Numeric - .subtract(Numeric.maxValue(decimalScale), Numeric.assertFromBigDecimal(decimalScale, 1)) - .value val maxUsd = Numeric - .multiply(decimalScale, maxDecimal, Numeric.assertFromBigDecimal(decimalScale, price)) + .multiply( + decimalScale, + Numeric.assertFromBigDecimal(decimalScale, tapLimit), + Numeric.assertFromBigDecimal(decimalScale, price), + ) .value // Integration test that the tap goes through aliceWalletClient.tap(maxUsd) val amulet = aliceValidatorBackend.participantClientWithAdminToken.ledger_api_extensions.acs .filterJava(amuletCodegen.Amulet.COMPANION)(aliceParty, _ => true) .loneElement - // Unit test that expiry does the right thing - SpliceUtil.amuletExpiresAt(amulet.data) shouldBe new Round(Long.MaxValue) - // Test that the USD/CC conversions get us to the max Decimal value ignoring decimal points + // Test that the USD/CC conversions get us to the limit ignoring decimal points amulet.data.amount.initialAmount.setScale(0, java.math.RoundingMode.DOWN) shouldBe Numeric - .maxValue(decimalScale) + .assertFromBigDecimal(decimalScale, tapLimit) .setScale(0, java.math.RoundingMode.DOWN) } diff --git a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/runbook/DsoPreflightIntegrationTest.scala b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/runbook/DsoPreflightIntegrationTest.scala index b42895f6ea..92d3f89ec9 100644 --- a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/runbook/DsoPreflightIntegrationTest.scala +++ b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/runbook/DsoPreflightIntegrationTest.scala @@ -25,11 +25,11 @@ class DsoPreflightIntegrationTest this.getClass.getSimpleName() ) - "SVs 1-3 + DA-1 are online and reachable via their public HTTP API" in { implicit env => + "SVs 1-3 + DA-1 report ready" in { implicit env => env.svs.remote.foreach(sv => clue(s"Checking SV at ${sv.httpClientConfig.url}") { eventuallySucceeds(timeUntilSuccess = 2.minutes) { - sv.getDsoInfo() + sv.httpReady shouldBe true } } ) @@ -74,7 +74,7 @@ class DsoPreflightIntegrationTest } } - "The Web UIs of SVs 1-3 + DA-1 are reachable and working as expected" in { env => + "The Web UIs of SVs 1-3 + DA-1 are reachable and working as expected" in { implicit env => // we put many checks in one test case to reduce testing time (logging in is slow) for ((svName, ingressName) <- coreSvIngressNames) { val svUiUrl = s"https://sv.${ingressName}.${sys.env("NETWORK_APPS_ADDRESS")}/"; @@ -82,18 +82,21 @@ class DsoPreflightIntegrationTest val svUsername = s"admin@${svName}-dev.com"; // our current practice is to use the same password for all SVs val svPassword = sys.env(s"SV_DEV_NET_WEB_UI_PASSWORD") - val sv = env.svs.remote.find(sv => sv.name == svName).value - val svInfo = eventuallySucceeds() { sv.getDsoInfo() } + // Each SV's party is read via its own scan (whose sv_party_id is that SV's party) + val svParty = eventuallySucceeds() { scancl(s"${svName}Scan").getDsoInfo().svParty } val votedSvParties = - env.svs.remote.filter(_ != sv).map(sv_ => eventuallySucceeds() { sv_.getDsoInfo().svParty }) + coreSvIngressNames.keys + .filter(_ != svName) + .toSeq + .map(other => eventuallySucceeds() { scancl(s"${other}Scan").getDsoInfo().svParty }) withFrontEnd("sv") { implicit webDriver => testSvUi( svUiUrl, svUsername, svPassword, - Some(svInfo), + Some(svParty), votedSvParties, ) } diff --git a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/runbook/RunbookSvPreflightIntegrationTestBase.scala b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/runbook/RunbookSvPreflightIntegrationTestBase.scala index 3ff56d669b..39cb172351 100644 --- a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/runbook/RunbookSvPreflightIntegrationTestBase.scala +++ b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/runbook/RunbookSvPreflightIntegrationTestBase.scala @@ -76,10 +76,10 @@ abstract class RunbookSvPreflightIntegrationTestBase } "The SV rewards are claimed by the SV, with 33.33% going to validator1" in { implicit env => - val svClient = sv_client("sv") + val svScanClient = scancl("svScan") val sv1ScanClient = scancl("sv1Scan") - val dsoInfo = svClient.getDsoInfo() + val dsoInfo = svScanClient.getDsoInfo() val svParty = dsoInfo.svParty.toProtoPrimitive val svInfo = dsoInfo.dsoRules.payload.svs.asScala.get(svParty).value val joinedAsOfRound = svInfo.joinedAsOfRound.number @@ -203,7 +203,7 @@ abstract class RunbookSvPreflightIntegrationTestBase ansAcronym, ) clue(s"Reserved ANS name can be looked up via scan") { - val svScanClient = scancl("svTestScan") + val svScanClient = scancl("svScan") eventuallySucceeds(3.minutes) { svScanClient.lookupEntryByName(ansName) } @@ -221,7 +221,7 @@ abstract class RunbookSvPreflightIntegrationTestBase )(noTracingLogger) } val svValidatorClient = vc("svTestValidator").copy(token = Some(token)) - val svScanClient = scancl("svTestScan") + val svScanClient = scancl("svScan") val sv1ScanClient = scancl("sv1Scan") val participantId = clue("Can dump participant identities from SV validator") { // retry to guard against badly timed restarts diff --git a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/runbook/RunbookSvSequencerInfoPreflightIntegrationTest.scala b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/runbook/RunbookSvSequencerInfoPreflightIntegrationTest.scala index 5224d2f2ff..3c568f7bc4 100644 --- a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/runbook/RunbookSvSequencerInfoPreflightIntegrationTest.scala +++ b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/runbook/RunbookSvSequencerInfoPreflightIntegrationTest.scala @@ -9,7 +9,9 @@ import scala.jdk.OptionConverters.* /** Preflight test that makes sure that the sequencer url is published to dsoRules */ -class RunbookSvSequencerInfoPreflightIntegrationTest extends IntegrationTest { +class RunbookSvSequencerInfoPreflightIntegrationTest + extends IntegrationTest + with PreflightIntegrationTestUtil { override lazy val resetRequiredTopologyState: Boolean = false override protected def runTokenStandardCliSanityCheck: Boolean = false @@ -20,9 +22,8 @@ class RunbookSvSequencerInfoPreflightIntegrationTest extends IntegrationTest { ) "The SV sequencer public url has been published to DsoRules" in { implicit env => - val sv = sv_client("sv") val dsoInfo = eventuallySucceeds() { - sv.getDsoInfo() + scancl("svScan").getDsoInfo() } val nodeState: SvNodeState = dsoInfo.svNodeStates.get(dsoInfo.svParty).value.payload val domainConfig = nodeState.state.synchronizerNodes.asScala.values.headOption.value diff --git a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/runbook/SvNonDevNetPreflightintegrationTest.scala b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/runbook/SvNonDevNetPreflightintegrationTest.scala index 3dfd7ac233..78f6cd9432 100644 --- a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/runbook/SvNonDevNetPreflightintegrationTest.scala +++ b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/runbook/SvNonDevNetPreflightintegrationTest.scala @@ -11,6 +11,7 @@ import java.time.Instant abstract class SvNonDevNetPreflightIntegrationTestBase extends FrontendIntegrationTest("sv") + with PreflightIntegrationTestUtil with SvUiPreflightIntegrationTestUtil with DataExportTestUtil with FrontendLoginUtil { @@ -35,7 +36,7 @@ abstract class SvNonDevNetPreflightIntegrationTestBase protected def svScanClient(implicit env: SpliceTestConsoleEnvironment) = scancl(s"${svName}Scan") "SV reports devnet=false" in { implicit env => - svClient.getDsoInfo().dsoRules.payload.isDevNet shouldBe false + svScanClient.getDsoInfo().dsoRules.payload.isDevNet shouldBe false } val svUsername = s"admin@${svName}.com" diff --git a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/runbook/SvUiPreflightIntegrationTestUtil.scala b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/runbook/SvUiPreflightIntegrationTestUtil.scala index 06778fd4a1..1504805c4a 100644 --- a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/runbook/SvUiPreflightIntegrationTestUtil.scala +++ b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/runbook/SvUiPreflightIntegrationTestUtil.scala @@ -2,7 +2,6 @@ package org.lfdecentralizedtrust.splice.integration.tests.runbook import org.lfdecentralizedtrust.splice.integration.tests.SpliceTests.TestCommon import org.lfdecentralizedtrust.splice.integration.tests.FrontendTestCommon -import org.lfdecentralizedtrust.splice.sv.admin.api.client.commands.HttpSvPublicAppClient.DsoInfo import com.digitalasset.canton.topology.PartyId import scala.concurrent.duration.* @@ -21,7 +20,7 @@ trait SvUiPreflightIntegrationTestUtil extends TestCommon { svUiUrl: String, svUsername: String, svPassword: String, - svInfo: Option[DsoInfo], + expectedSvParty: Option[PartyId], votedSvParties: Seq[PartyId], extraChecks: => Unit = (), )(implicit webDriver: WebDriverType) = { @@ -39,11 +38,10 @@ trait SvUiPreflightIntegrationTestUtil extends TestCommon { )( s"We see a table with correct info data about the SV", _ => { - svInfo.foreach(si => + expectedSvParty.foreach(party => inside(findAll(className("general-dso-value-name")).toSeq.take(2)) { - case Seq(svUser, svPartyId) => - seleniumText(svUser) should matchText(si.svUser) - seleniumText(svPartyId) should matchText(si.svParty.toProtoPrimitive) + case Seq(_, svPartyId) => + seleniumText(svPartyId) should matchText(party.toProtoPrimitive) } ) }, diff --git a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/runbook/ValidatorPreflightIntegrationTest.scala b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/runbook/ValidatorPreflightIntegrationTest.scala index a07d90536c..817dbaf3a9 100644 --- a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/runbook/ValidatorPreflightIntegrationTest.scala +++ b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/runbook/ValidatorPreflightIntegrationTest.scala @@ -626,9 +626,8 @@ class RunbookValidatorPreflightIntegrationTest extends ValidatorPreflightIntegra // TODO(#979): remove this check once canton handles sequencer connections more gracefully override def checkValidatorIsConnectedToSvRunbook() = "Validator is connected to SV runbook" in { implicit env => - val sv = sv_client("sv") eventually(2.minutes) { - val dsoInfo = sv.getDsoInfo() + val dsoInfo = scancl("svScan").getDsoInfo() val nodeState = dsoInfo.svNodeStates.get(dsoInfo.svParty).value.payload val synchronizerNodeConfig = nodeState.state.synchronizerNodes.asScala.values.headOption.value diff --git a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/util/UpdateHistoryTestUtil.scala b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/util/UpdateHistoryTestUtil.scala index cd1d00c8af..501f34cb73 100644 --- a/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/util/UpdateHistoryTestUtil.scala +++ b/apps/app/src/test/scala/org/lfdecentralizedtrust/splice/util/UpdateHistoryTestUtil.scala @@ -33,7 +33,12 @@ import org.lfdecentralizedtrust.splice.store.UpdateHistoryTestBase.{ LostInScanApi, LostInStoreIngestion, } -import org.lfdecentralizedtrust.splice.store.{PageLimit, UpdateHistory, UpdateHistoryTestBase} +import org.lfdecentralizedtrust.splice.store.{ + PageLimit, + TimestampWithMigrationId, + UpdateHistory, + UpdateHistoryTestBase, +} import org.lfdecentralizedtrust.splice.store.UpdateHistory.UpdateHistoryResponse import com.daml.ledger.api.v2.transaction_filter import com.digitalasset.canton.admin.api.client.commands.LedgerApiCommands.UpdateService.{ @@ -115,11 +120,11 @@ trait UpdateHistoryTestUtil extends TestCommon { val recordedUpdates = updateHistory .getAllUpdates( Some( - ( - 0L, + TimestampWithMigrationId( // The after0 argument to getUpdates() is exclusive, so we need to subtract a small value // to include the first element actualUpdates.head.update.recordTime.addMicros(-1L), + 0L, ) ), PageLimit.tryCreate(actualUpdates.size), @@ -140,11 +145,11 @@ trait UpdateHistoryTestUtil extends TestCommon { val recordedUpdates = updateHistory .getAllUpdates( Some( - ( - 0L, + TimestampWithMigrationId( // The after0 argument to getUpdates() is exclusive, so we need to subtract a small value // to include the first element actualUpdates.head.update.recordTime.addMicros(-1L), + 0L, ) ), PageLimit.tryCreate(actualUpdates.size), diff --git a/apps/common/frontend-test-handlers/src/mocks/handlers/dso-info-handler.ts b/apps/common/frontend-test-handlers/src/mocks/handlers/dso-info-handler.ts index 796bca9832..e1b08b486c 100644 --- a/apps/common/frontend-test-handlers/src/mocks/handlers/dso-info-handler.ts +++ b/apps/common/frontend-test-handlers/src/mocks/handlers/dso-info-handler.ts @@ -593,8 +593,9 @@ export const dsoInfo = { ], }; -export function dsoInfoHandler(baseUrl: string): HttpHandler { - return http.get(`${baseUrl}/v0/dso`, () => { +// The scan app serves DSO info at /v0/dso, the SV app at /v1/dso. +export function dsoInfoHandler(baseUrl: string, path: string = '/v0/dso'): HttpHandler { + return http.get(`${baseUrl}${path}`, () => { return HttpResponse.json(dsoInfo); }); } diff --git a/apps/common/frontend/src/contexts/SvServiceContext.tsx b/apps/common/frontend/src/contexts/SvServiceContext.tsx index 5f909943fe..ae62bf1c0b 100644 --- a/apps/common/frontend/src/contexts/SvServiceContext.tsx +++ b/apps/common/frontend/src/contexts/SvServiceContext.tsx @@ -1,8 +1,19 @@ // Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. // SPDX-License-Identifier: Apache-2.0 import * as openapi from '@canton-network/sv-openapi'; +import { useUserState } from '@canton-network/splice-common-frontend'; +import { + BaseApiMiddleware, + OpenAPILoggingMiddleware, +} from '@canton-network/splice-common-frontend-utils'; import React, { useContext, useMemo } from 'react'; -import { GetDsoInfoResponse, ServerConfiguration } from '@canton-network/sv-openapi'; +import { + GetDsoInfoResponse, + Middleware, + RequestContext, + ResponseContext, + ServerConfiguration, +} from '@canton-network/sv-openapi'; const SvContext = React.createContext(undefined); @@ -14,19 +25,26 @@ export interface SvClient { getDsoInfo: () => Promise; } +class ApiMiddleware + extends BaseApiMiddleware + implements Middleware {} + export const SvClientProvider: React.FC> = ({ url, children }) => { + const { userAccessToken } = useUserState(); + const friendlyClient: SvClient | undefined = useMemo(() => { const configuration = openapi.createConfiguration({ baseServer: new ServerConfiguration(url, {}), + promiseMiddleware: [new ApiMiddleware(userAccessToken), new OpenAPILoggingMiddleware('sv')], }); const svClient = new openapi.SvApi(configuration); return { getDsoInfo: async (): Promise => { - return await svClient.getDsoInfo(); + return await svClient.getDsoInfoV1(); }, }; - }, [url]); + }, [url, userAccessToken]); return {children}; }; diff --git a/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/admin/api/HttpRequestLogger.scala b/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/admin/api/HttpRequestLogger.scala index b5514fdeb6..364d1fe413 100644 --- a/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/admin/api/HttpRequestLogger.scala +++ b/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/admin/api/HttpRequestLogger.scala @@ -4,12 +4,13 @@ package org.lfdecentralizedtrust.splice.admin.api import org.apache.pekko.http.scaladsl.model.{ContentTypes, HttpEntity, RemoteAddress} -import org.apache.pekko.http.scaladsl.server.{Directive0, RequestContext} +import org.apache.pekko.http.scaladsl.server.{Directive0, Directive1, RequestContext} import org.apache.pekko.http.scaladsl.server.Directives.* import com.digitalasset.canton.config.ApiLoggingConfig import com.digitalasset.canton.logging.{NamedLoggerFactory, NamedLogging} import com.digitalasset.canton.tracing.TraceContext import com.digitalasset.canton.util.ShowUtil.* +import org.lfdecentralizedtrust.splice.http.ClientIpDirectives object HttpRequestLogger { def apply( @@ -17,6 +18,7 @@ object HttpRequestLogger { maxPathLength: Int, maxStringLength: Int, maxMetadataSize: Int, + clientIpHeaders: Seq[String], loggerFactory: NamedLoggerFactory, )(implicit traceContext: TraceContext): Directive0 = { new HttpRequestLogger( @@ -24,6 +26,7 @@ object HttpRequestLogger { maxPathLength, maxStringLength, maxMetadataSize, + clientIpHeaders, loggerFactory, ).directive } @@ -31,12 +34,14 @@ object HttpRequestLogger { // ignores maxMethodLength and maxMessageLines def apply( loggingConfig: ApiLoggingConfig, + clientIpHeaders: Seq[String], loggerFactory: NamedLoggerFactory, )(implicit traceContext: TraceContext): Directive0 = apply( messagePayloads = loggingConfig.messagePayloads, maxPathLength = loggingConfig.maxMethodLength, maxStringLength = loggingConfig.maxStringLength, maxMetadataSize = loggingConfig.maxMetadataSize, + clientIpHeaders = clientIpHeaders, loggerFactory = loggerFactory, ) } @@ -46,6 +51,7 @@ final class HttpRequestLogger( maxPathLength: Int, maxStringLength: Int, maxMetadataSize: Int, + clientIpHeaders: Seq[String], override protected val loggerFactory: NamedLoggerFactory, ) extends NamedLogging { def createLogMessage(ctx: RequestContext, remoteAddress: RemoteAddress)( @@ -56,8 +62,14 @@ final class HttpRequestLogger( s"HTTP ${ctx.request.method.name} ${pathLimited} from (${remoteAddress}): ${message}" } + private def extractConfiguredClientIp: Directive1[RemoteAddress] = + ClientIpDirectives.extractClientIp(clientIpHeaders).flatMap { + case Some(remoteAddress) => provide(remoteAddress) + case None => extractClientIP + } + private def directive(implicit traceContext: TraceContext): Directive0 = { - extractClientIP.flatMap { remoteAddress => + extractConfiguredClientIp.flatMap { remoteAddress => extractRequestContext.flatMap { ctx => val msg = createLogMessage(ctx, remoteAddress) logger.debug(msg("received request.")) diff --git a/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/admin/http/HttpAdminService.scala b/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/admin/http/HttpAdminService.scala index 3494946147..04bdee6c17 100644 --- a/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/admin/http/HttpAdminService.scala +++ b/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/admin/http/HttpAdminService.scala @@ -36,6 +36,7 @@ object HttpAdminService { adminApi: AdminServerConfig, parameterConfig: CantonNodeParameters, apiLoggingConfig: ApiLoggingConfig, + clientIpHeaders: Seq[String], loggerFactory: NamedLoggerFactory, node: => Option[CantonNode], )(implicit @@ -49,6 +50,7 @@ object HttpAdminService { adminApi.port, parameterConfig, apiLoggingConfig, + clientIpHeaders, loggerFactory, node, ) @@ -59,6 +61,7 @@ object HttpAdminService { port: Port, parameterConfig: CantonNodeParameters, apiLoggingConfig: ApiLoggingConfig, + clientIpHeaders: Seq[String], loggerFactory: NamedLoggerFactory, node: => Option[CantonNode], )(implicit ac: ActorSystem, ec: ExecutionContext, tracer: Tracer, elc: ErrorLoggingContext) @@ -98,7 +101,7 @@ object HttpAdminService { // handleRejections (inside the logger) seals the route: rejections are // converted to HTTP responses so mapResponse sees all outcomes and logs // exactly one "Responding with status code" per request. - HttpRequestLogger(apiLoggingConfig, loggerFactory)(traceContext) { + HttpRequestLogger(apiLoggingConfig, clientIpHeaders, loggerFactory)(traceContext) { handleRejections(RejectionHandler.default) { encodeResponse( handleRejections( diff --git a/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/config/SpliceConfig.scala b/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/config/SpliceConfig.scala index f433f9b66c..3ee52a5994 100644 --- a/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/config/SpliceConfig.scala +++ b/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/config/SpliceConfig.scala @@ -25,6 +25,7 @@ abstract class SpliceBackendConfig extends LocalNodeConfig { def participantClient: BaseParticipantClientConfig def automation: AutomationConfig + def parameters: SpliceParametersConfig } diff --git a/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/environment/NodeBootstrapBase.scala b/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/environment/NodeBootstrapBase.scala index 729411f223..bb6fa45add 100644 --- a/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/environment/NodeBootstrapBase.scala +++ b/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/environment/NodeBootstrapBase.scala @@ -7,7 +7,7 @@ import cats.data.EitherT import com.daml.nameof.NameOf.functionFullName import org.lfdecentralizedtrust.splice.SpliceMetrics import com.digitalasset.canton.concurrent.ExecutionContextIdlenessExecutorService -import com.digitalasset.canton.config.{LocalNodeConfig, ProcessingTimeout} +import com.digitalasset.canton.config.ProcessingTimeout import com.digitalasset.canton.config.CantonRequireTypes.InstanceName import com.digitalasset.canton.crypto.Crypto import com.digitalasset.canton.environment.{CantonNode, CantonNodeBootstrap, CantonNodeParameters} @@ -26,6 +26,7 @@ import java.util.concurrent.atomic.{AtomicBoolean, AtomicReference} import scala.concurrent.{blocking, Future} import scala.util.{Failure, Success} import org.lfdecentralizedtrust.splice.admin.http.{AdminRoutes, HttpAdminService} +import org.lfdecentralizedtrust.splice.config.SpliceBackendConfig /** Modelled after CantonNodeBootstrap */ @@ -66,7 +67,7 @@ trait NodeBootstrap[+N <: CantonNode] */ abstract class NodeBootstrapBase[ T <: CantonNode, - NodeConfig <: LocalNodeConfig, + NodeConfig <: SpliceBackendConfig, ParameterConfig <: CantonNodeParameters, ]( nodeConfig: NodeConfig, @@ -109,6 +110,7 @@ abstract class NodeBootstrapBase[ nodeConfig.adminApi, parameterConfig, parameterConfig.loggingConfig.api, + nodeConfig.parameters.rateLimiting.clientIpHeaders, loggerFactory, getNode, ) diff --git a/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/metrics/ScanConnectionMetrics.scala b/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/metrics/ScanConnectionMetrics.scala index a4ac00a1f1..a388fab313 100644 --- a/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/metrics/ScanConnectionMetrics.scala +++ b/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/metrics/ScanConnectionMetrics.scala @@ -38,7 +38,9 @@ class ScanConnectionMetrics(metricsFactory: LabeledMetricsFactory) { summary = "Count of succeeded and failed requests to a scan connection", qualification = Traffic, labelsWithDescription = perConnectionLabels ++ Map( - "outcome" -> "Category of failure or success" + "outcome" -> "Category of failure or success", + "http_status" -> ("For failures, the HTTP status code of the response when available, " + + "'none' otherwise (e.g. transport-level failures). Absent for successful requests."), ), ) ) diff --git a/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/store/HistoryMetrics.scala b/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/store/HistoryMetrics.scala index 73f314d99f..7bdb6fe322 100644 --- a/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/store/HistoryMetrics.scala +++ b/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/store/HistoryMetrics.scala @@ -438,11 +438,15 @@ class HistoryMetrics(metricsFactory: LabeledMetricsFactory)(implicit ) )(metricsContext) - def incAcsSnapshotObjects(encoding: String): Unit = - objectsCount.inc()(MetricsContext("object_type" -> "ACS_snapshots", "encoding" -> encoding)) + def incAcsSnapshotObjects(encoding: String, bucket: String): Unit = + objectsCount.inc()( + MetricsContext("object_type" -> "ACS_snapshots", "encoding" -> encoding, "bucket" -> bucket) + ) - def incUpdateObjects(encoding: String): Unit = - objectsCount.inc()(MetricsContext("object_type" -> "updates", "encoding" -> encoding)) + def incUpdateObjects(encoding: String, bucket: String): Unit = + objectsCount.inc()( + MetricsContext("object_type" -> "updates", "encoding" -> encoding, "bucket" -> bucket) + ) def incUpdatesCount(count: Int): Unit = updatesCount.inc(count.toLong)(MetricsContext.Empty) diff --git a/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/store/S3BucketConnection.scala b/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/store/S3BucketConnection.scala index fe992c363b..66b0c89c50 100644 --- a/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/store/S3BucketConnection.scala +++ b/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/store/S3BucketConnection.scala @@ -222,6 +222,14 @@ class S3BucketConnection( private val parts = TrieMap.empty[Integer, CompletedPart] private val md = MessageDigest.getInstance("SHA-256") + /** The checksum of the whole object. Computing it via a `lazy val` to support idempotent `finish()` calls. + */ + private lazy val objectChecksum: String = Base64.getEncoder.encodeToString(md.digest()) + + /** `lazy val` to ensure that multi-part upload is completed at most once. + */ + private lazy val finishResult: Future[Unit] = doFinish() + /** Call this once before uploading a new part. * The content must already be provided for checksums, but will not be uploaded yet. */ @@ -273,7 +281,12 @@ class S3BucketConnection( } } - def finish(): Future[Unit] = { + /** Completes the multi-part upload and stores the object checksum in the object's metadata. + * Idempotent, safe to call more than once (will just return the Future from the first call again). + */ + def finish(): Future[Unit] = finishResult + + private def doFinish(): Future[Unit] = { require(numParts.get() > 0) require( parts.size == numParts.get(), @@ -297,7 +310,7 @@ class S3BucketConnection( _ <- s3Client.completeMultipartUpload(completeRequest).asScala // Copy-in-place of the object to add the final checksum to its metadata - metadata = Map("splice-checksum" -> Base64.getEncoder.encodeToString(md.digest())) + metadata = Map("splice-checksum" -> objectChecksum) copyReq = CopyObjectRequest .builder() diff --git a/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/store/UpdateHistory.scala b/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/store/UpdateHistory.scala index db3161ae2e..3554789a01 100644 --- a/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/store/UpdateHistory.scala +++ b/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/store/UpdateHistory.scala @@ -40,6 +40,8 @@ import com.digitalasset.canton.config.CantonRequireTypes.String256M import com.digitalasset.canton.data.CantonTimestamp import com.digitalasset.canton.lifecycle.CloseContext import com.digitalasset.canton.logging.{NamedLoggerFactory, NamedLogging} +import com.digitalasset.canton.logging.pretty.Pretty +import com.digitalasset.canton.logging.pretty.Pretty.{param, prettyOfClass} import com.digitalasset.canton.resource.DbStorage import com.digitalasset.canton.topology.{ParticipantId, PartyId, SynchronizerId} import com.digitalasset.canton.tracing.TraceContext @@ -906,14 +908,14 @@ class UpdateHistory( } private def afterFilters( - afterO: Option[(Long, CantonTimestamp)], + afterO: Option[TimestampWithMigrationId], includeImportUpdates: Boolean, ): NonEmptyList[SQLActionBuilder] = { val gtMin = if (includeImportUpdates) ">=" else ">" afterO match { case None => NonEmptyList.of(sql"migration_id >= 0 and record_time #$gtMin ${CantonTimestamp.MinValue}") - case Some((afterMigrationId, afterRecordTime)) => + case Some(TimestampWithMigrationId(afterRecordTime, afterMigrationId)) => // This makes it so that the two queries use updt_hist_tran_hi_mi_rt_di, NonEmptyList.of( sql"migration_id = ${afterMigrationId} and record_time > ${afterRecordTime} ", @@ -1101,7 +1103,7 @@ class UpdateHistory( } def getUpdatesWithoutImportUpdates( - afterO: Option[(Long, CantonTimestamp)], + afterO: Option[TimestampWithMigrationId], limit: Limit, )(implicit tc: TraceContext): Future[Seq[TreeUpdateWithMigrationId]] = { val filters = afterFilters(afterO, includeImportUpdates = false) @@ -1132,7 +1134,7 @@ class UpdateHistory( } def getAllUpdates( - afterO: Option[(Long, CantonTimestamp)], + afterO: Option[TimestampWithMigrationId], limit: PageLimit, )(implicit tc: TraceContext): Future[Seq[TreeUpdateWithMigrationId]] = { val filters = afterFilters(afterO, includeImportUpdates = true) @@ -2595,6 +2597,12 @@ final case class TimestampWithMigrationId( object TimestampWithMigrationId { implicit val ordering: Ordering[TimestampWithMigrationId] = Ordering.by(x => (x.migrationId, x.timestamp)) + + implicit val prettyTimestampWithMigrationId: Pretty[TimestampWithMigrationId] = + prettyOfClass( + param("timestamp", _.timestamp), + param("migrationId", _.migrationId), + ) } final case class TreeUpdateWithMigrationId( diff --git a/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/util/DsoInfo.scala b/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/util/DsoInfo.scala new file mode 100644 index 0000000000..703ae52ae6 --- /dev/null +++ b/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/util/DsoInfo.scala @@ -0,0 +1,85 @@ +// Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package org.lfdecentralizedtrust.splice.util + +import cats.syntax.either.* +import cats.syntax.traverse.* +import com.digitalasset.canton.logging.ErrorLoggingContext +import org.lfdecentralizedtrust.splice.codegen.java.splice.amuletrules.AmuletRules +import org.lfdecentralizedtrust.splice.codegen.java.splice.dso.svstate.SvNodeState +import org.lfdecentralizedtrust.splice.codegen.java.splice.dsorules.DsoRules +import org.lfdecentralizedtrust.splice.codegen.java.splice.round.OpenMiningRound +import org.lfdecentralizedtrust.splice.http.v0.definitions +import org.lfdecentralizedtrust.splice.util.{ContractWithState, TemplateJsonDecoder} +import com.digitalasset.canton.topology.PartyId + +/** Decoded version of [[definitions.GetDsoInfoResponse]], as served by scan's `/v0/dso` + * and the SV app's `/v1/dso` endpoints. + */ +final case class DsoInfo( + svUser: String, + svParty: PartyId, + dsoParty: PartyId, + votingThreshold: BigInt, + latestMiningRound: ContractWithState[OpenMiningRound.ContractId, OpenMiningRound], + amuletRules: ContractWithState[AmuletRules.ContractId, AmuletRules], + dsoRules: ContractWithState[DsoRules.ContractId, DsoRules], + svNodeStates: Map[PartyId, ContractWithState[SvNodeState.ContractId, SvNodeState]], + initialRound: Option[String], +) { + + def toHttp(implicit elc: ErrorLoggingContext): definitions.GetDsoInfoResponse = + definitions.GetDsoInfoResponse( + svUser, + svParty.toProtoPrimitive, + dsoParty.toProtoPrimitive, + votingThreshold, + latestMiningRound.toHttp, + amuletRules.toHttp, + dsoRules.toHttp, + svNodeStates.values.map(_.toHttp).toVector, + initialRound, + ) +} + +object DsoInfo { + + def fromHttp( + dsoInfo: definitions.GetDsoInfoResponse + )(implicit decoder: TemplateJsonDecoder): Either[String, DsoInfo] = + for { + svPartyId <- Codec.decode(Codec.Party)(dsoInfo.svPartyId) + dsoPartyId <- Codec.decode(Codec.Party)(dsoInfo.dsoPartyId) + latestMiningRound <- ContractWithState + .fromHttp(OpenMiningRound.COMPANION)(dsoInfo.latestMiningRound) + .leftMap(_.toString) + amuletRules <- ContractWithState + .fromHttp(AmuletRules.COMPANION)(dsoInfo.amuletRules) + .left + .map(_.toString) + dsoRules <- ContractWithState + .fromHttp(DsoRules.COMPANION)(dsoInfo.dsoRules) + .left + .map(_.toString) + svNodeStates <- dsoInfo.svNodeStates.traverse { co => + for { + nodeState <- ContractWithState + .fromHttp(SvNodeState.COMPANION)(co) + .left + .map(_.toString) + partyId <- Codec.decode(Codec.Party)(nodeState.payload.sv) + } yield partyId -> nodeState + } + } yield DsoInfo( + dsoInfo.svUser, + svPartyId, + dsoPartyId, + dsoInfo.votingThreshold, + latestMiningRound, + amuletRules, + dsoRules, + svNodeStates.toMap, + dsoInfo.initialRound, + ) +} diff --git a/apps/common/src/test/scala/org/lfdecentralizedtrust/splice/admin/api/HttpRequestLoggerTest.scala b/apps/common/src/test/scala/org/lfdecentralizedtrust/splice/admin/api/HttpRequestLoggerTest.scala index 2037a0cc93..0f8121f2fe 100644 --- a/apps/common/src/test/scala/org/lfdecentralizedtrust/splice/admin/api/HttpRequestLoggerTest.scala +++ b/apps/common/src/test/scala/org/lfdecentralizedtrust/splice/admin/api/HttpRequestLoggerTest.scala @@ -6,19 +6,22 @@ package org.lfdecentralizedtrust.splice.admin.api import com.digitalasset.canton.config.ApiLoggingConfig import com.digitalasset.canton.logging.SuppressionRule import com.digitalasset.canton.BaseTest -import org.apache.pekko.http.scaladsl.model.StatusCodes +import org.apache.pekko.http.scaladsl.model.{HttpRequest, RemoteAddress, StatusCodes} +import org.apache.pekko.http.scaladsl.model.headers.{RawHeader, `X-Forwarded-For`} import org.apache.pekko.http.scaladsl.server.{RejectionHandler, Route} import org.apache.pekko.http.scaladsl.server.Directives.* import org.apache.pekko.http.scaladsl.testkit.ScalatestRouteTest import org.scalatest.wordspec.AnyWordSpec import org.slf4j.event.Level +import org.lfdecentralizedtrust.splice.config.RateLimitersConfig class HttpRequestLoggerTest extends AnyWordSpec with BaseTest with ScalatestRouteTest { private val apiLoggingConfig = ApiLoggingConfig() - private def loggerDirective = HttpRequestLogger( + private def loggerDirective(clientIpHeaders: Seq[String]) = HttpRequestLogger( apiLoggingConfig, + clientIpHeaders, loggerFactory, ) @@ -28,8 +31,10 @@ class HttpRequestLoggerTest extends AnyWordSpec with BaseTest with ScalatestRout * all outcomes and logs exactly one "Responding with status code" per request — * whether matched or rejected. */ - private def route: Route = - loggerDirective { + private def routeFor( + clientIpHeaders: Seq[String] = RateLimitersConfig.DefaultClientIpHeaders + ): Route = + loggerDirective(clientIpHeaders) { handleRejections(RejectionHandler.default) { concat( pathPrefix("api" / "admin") { @@ -42,8 +47,44 @@ class HttpRequestLoggerTest extends AnyWordSpec with BaseTest with ScalatestRout } } + private lazy val route = routeFor() + + private def assertLoggedClientIp(request: HttpRequest, expectedClientIp: String): Unit = + loggerFactory.assertLogsSeq(SuppressionRule.Level(Level.DEBUG))( + { + request ~> routeFor(Seq("x-envoy-external-address")) ~> check { + status shouldBe StatusCodes.OK + } + }, + logEntries => + forExactly(1, logEntries) { entry => + entry.message should include("received request") + entry.message should include(s"from ($expectedClientIp)") + }, + ) + "HttpRequestLogger" should { + "prefer the configured client IP header" in { + assertLoggedClientIp( + Get("/api/app").withHeaders( + RawHeader("X-Envoy-External-Address", "5.5.5.5"), + RawHeader("X-Forwarded-For", "1.1.1.1"), + ), + "5.5.5.5", + ) + } + + "fall back to the existing client IP extraction" in { + assertLoggedClientIp( + Get("/api/app").withHeaders( + RawHeader("X-Envoy-External-Address", "not-an-ip"), + `X-Forwarded-For`(Seq(RemoteAddress(Array[Byte](2, 2, 2, 2)))), + ), + "2.2.2.2", + ) + } + "log exactly one 'received request' and one response for a matching route" in { loggerFactory.assertLogsSeq(SuppressionRule.Level(Level.DEBUG))( { @@ -84,7 +125,7 @@ class HttpRequestLoggerTest extends AnyWordSpec with BaseTest with ScalatestRout "one log entry per request when methods conflict across siblings" in { val newStyleMethodRoute: Route = - loggerDirective { + loggerDirective(RateLimitersConfig.DefaultClientIpHeaders) { handleRejections(RejectionHandler.default) { concat( pathPrefix("api" / "data") { diff --git a/apps/common/src/test/scala/org/lfdecentralizedtrust/splice/store/UpdateHistoryTest.scala b/apps/common/src/test/scala/org/lfdecentralizedtrust/splice/store/UpdateHistoryTest.scala index 7e8cf83157..112279eb34 100644 --- a/apps/common/src/test/scala/org/lfdecentralizedtrust/splice/store/UpdateHistoryTest.scala +++ b/apps/common/src/test/scala/org/lfdecentralizedtrust/splice/store/UpdateHistoryTest.scala @@ -424,7 +424,10 @@ class UpdateHistoryTest extends UpdateHistoryTestBase { store .getAllUpdates( after.map { case (migrationId, recordTime) => - (migrationId, CantonTimestamp.assertFromInstant(recordTime)) + TimestampWithMigrationId( + CantonTimestamp.assertFromInstant(recordTime), + migrationId, + ) }, PageLimit.tryCreate(1), ) diff --git a/apps/metrics-docs/src/main/scala/org/lfdecentralizedtrust/splice/metrics/MetricsDocs.scala b/apps/metrics-docs/src/main/scala/org/lfdecentralizedtrust/splice/metrics/MetricsDocs.scala index ba957e162b..68e96ebc2b 100644 --- a/apps/metrics-docs/src/main/scala/org/lfdecentralizedtrust/splice/metrics/MetricsDocs.scala +++ b/apps/metrics-docs/src/main/scala/org/lfdecentralizedtrust/splice/metrics/MetricsDocs.scala @@ -32,7 +32,7 @@ import org.lfdecentralizedtrust.splice.sv.automation.confirmation.{ } import org.lfdecentralizedtrust.splice.sv.automation.delegatebased.ProcessRewardsTriggerBase import org.lfdecentralizedtrust.splice.validator.metrics.TopologyMetrics -import org.lfdecentralizedtrust.splice.wallet.metrics.AmuletMetrics +import org.lfdecentralizedtrust.splice.wallet.metrics.{AmuletMetrics, TreasuryMetrics} final case class GeneratedMetrics( common: List[MetricDoc.Item], @@ -96,6 +96,7 @@ object MetricsDocs { generator.reset() // validator new AmuletMetrics(walletUserParty, generator) + new TreasuryMetrics(walletUserParty, generator, () => 0L) val topologyMetrics = new TopologyMetrics(generator) // force creation of a gauge for a dummy participant val _ = topologyMetrics.getNumPartiesPerParticipantGauge( diff --git a/apps/scan/src/main/openapi/scan.yaml b/apps/scan/src/main/openapi/scan.yaml index f5513b5d87..efbc94a9f4 100644 --- a/apps/scan/src/main/openapi/scan.yaml +++ b/apps/scan/src/main/openapi/scan.yaml @@ -638,10 +638,14 @@ paths: /v1/state/acs: post: + deprecated: true tags: [ external, scan ] x-jvm-package: scan operationId: "getAcsSnapshotAtV1" description: | + Deprecated. Please use /v2/state/acs instead. + The only difference with this endpoint and that one is the type of the `after`/`next_page_token` pagination token. + Returns the ACS in creation date ascending order, paged, for a given migration id and record time. Unlike /v0/state/acs, every contract is identified by an (optional) update_id (as opposed to the event ID in /v0/state/acs, which was not BFT-safe). @@ -667,6 +671,37 @@ paths: "500": $ref: "../../../../common/src/main/openapi/common-external.yaml#/components/responses/500" + /v2/state/acs: + post: + tags: [ external, scan ] + x-jvm-package: scan + operationId: "getAcsSnapshotAtV2" + description: | + Returns the ACS in creation date ascending order, paged, for a given migration id and record time. + Unlike /v0/state/acs, every contract is identified by an (optional) update_id + (as opposed to the event ID in /v0/state/acs, which was not BFT-safe). + The update_id is the ID of the update in which the contract was created, and can be used to correlate with updates returned by /v2/updates. + For contracts created in an earlier migration ID, the update_id will be absent. + requestBody: + required: true + content: + application/json: + schema: + "$ref": "#/components/schemas/AcsRequestV2" + responses: + "200": + description: ok + content: + application/json: + schema: + $ref: "#/components/schemas/AcsResponseV2" + "400": + $ref: "../../../../common/src/main/openapi/common-external.yaml#/components/responses/400" + "404": + $ref: "../../../../common/src/main/openapi/common-external.yaml#/components/responses/404" + "500": + $ref: "../../../../common/src/main/openapi/common-external.yaml#/components/responses/500" + /v0/state/acs/force: post: tags: [external, scan] @@ -719,10 +754,14 @@ paths: /v1/holdings/state: post: + deprecated: true tags: [external, scan] x-jvm-package: scan operationId: "getHoldingsStateAtV1" description: | + Deprecated. Please use /v2/holdings/state instead. + The only difference with this endpoint and that one is the type of the `after`/`next_page_token` pagination token. + Returns the active amulet contracts for a given migration id and record time, in creation date ascending order, paged. requestBody: required: true @@ -744,6 +783,33 @@ paths: "500": $ref: "../../../../common/src/main/openapi/common-external.yaml#/components/responses/500" + /v2/holdings/state: + post: + tags: [external, scan] + x-jvm-package: scan + operationId: "getHoldingsStateAtV2" + description: | + Returns the active amulet contracts for a given migration id and record time, in creation date ascending order, paged. + requestBody: + required: true + content: + application/json: + schema: + "$ref": "#/components/schemas/HoldingsStateRequestV2" + responses: + "200": + description: ok + content: + application/json: + schema: + $ref: "#/components/schemas/AcsResponseV2" + "400": + $ref: "../../../../common/src/main/openapi/common-external.yaml#/components/responses/400" + "404": + $ref: "../../../../common/src/main/openapi/common-external.yaml#/components/responses/404" + "500": + $ref: "../../../../common/src/main/openapi/common-external.yaml#/components/responses/500" + /v0/holdings/summary: post: deprecated: true @@ -2831,6 +2897,60 @@ components: type: string description: | Filters the ACS by contracts with these template IDs, specified as "PACKAGE_NAME:MODULE_NAME:ENTITY_NAME". + AcsRequestV2: + type: object + required: + - migration_id + - record_time + - page_size + properties: + migration_id: + type: integer + format: int64 + description: | + The migration id for which to return the ACS. + record_time: + type: string + format: date-time + description: | + The timestamp at which the contract set was active. + This needs to be an exact timestamp, i.e., + needs to correspond to a timestamp reported by `/v0/state/acs/snapshot-timestamp` if `record_time_match` is set to `exact` (which is the default). + If `record_time_match` is set to `at_or_before`, this can be any timestamp, and the most recent snapshot at or before the given `record_time` will be returned. + record_time_match: + type: string + description: | + How to match the record_time. "exact" requires the record_time to match exactly. + "at_or_before" finds the most recent snapshot at or before the given record_time. + enum: + - "exact" + - "at_or_before" + default: "exact" + after: + type: string + description: | + Pagination token for the next page of results. For this to be valid, + this must be the `next_page_token` from a prior request with identical + parameters aside from `after` and `page_size`; the response may be + invalid otherwise. + This token is opaque and not meant to be edited by users. + page_size: + description: | + The maximum number of created events returned for this request. + type: integer + format: int32 + party_ids: + type: array + items: + type: string + description: | + Filters the ACS by contracts in which these party IDs are stakeholders. + templates: + type: array + items: + type: string + description: | + Filters the ACS by contracts with these template IDs, specified as "PACKAGE_NAME:MODULE_NAME:ENTITY_NAME". HoldingsStateRequest: # subset of AcsRequest type: object required: @@ -2879,6 +2999,54 @@ components: description: | Filters by contracts in which these party_ids are the owners of the amulets. + HoldingsStateRequestV2: # subset of AcsRequestV2 + type: object + required: + - migration_id + - record_time + - page_size + - owner_party_ids + properties: + migration_id: + type: integer + format: int64 + description: | + The migration id for which to return the ACS. + record_time: + type: string + format: date-time + description: | + The timestamp at which the contract set was active. + This needs to be an exact timestamp, i.e., + needs to correspond to a timestamp reported by `/v0/state/acs/snapshot-timestamp` if `record_time_match` is set to `exact` (which is the default). + If `record_time_match` is set to `at_or_before`, this can be any timestamp, and the most recent snapshot at or before the given `record_time` will be returned. + record_time_match: + type: string + description: | + How to match the record_time. "exact" requires the record_time to match exactly. + "at_or_before" finds the most recent snapshot at or before the given record_time. + enum: + - "exact" + - "at_or_before" + default: "exact" + after: + type: string + description: | + Pagination token for the next page of results. + This token is opaque and not meant to be edited by users. + page_size: + description: | + The maximum number of created events returned for this request. + type: integer + format: int32 + owner_party_ids: + type: array + items: + type: string + minItems: 1 + description: | + Filters by contracts in which these party_ids are the owners of the amulets. + HoldingsSummaryRequest: type: object required: @@ -3033,6 +3201,36 @@ components: to the `AcsRequest` or `HoldingsStateRequest`. Will be absent when there are no more pages. + AcsResponseV2: + type: object + required: + - record_time + - migration_id + - created_events + properties: + record_time: + description: The same `record_time` as in the request. + type: string + format: date-time + migration_id: + description: The same `migration_id` as in the request. + type: integer + format: int64 + created_events: + description: | + Up to `page_size` contracts in the ACS. + `create_arguments` are always encoded as `compact_json`. + type: array + items: + $ref: "#/components/schemas/ActiveContract" + next_page_token: + type: string + description: | + When requesting the next page of results, pass this as `after` + to the `AcsRequestV2` or `HoldingsStateRequestV2`. + Will be absent when there are no more pages. + This token is opaque and not meant to be edited by users. + HoldingsSummaryResponse: type: object required: diff --git a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/ScanApp.scala b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/ScanApp.scala index 0999b6cae0..b07eb2f3b6 100644 --- a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/ScanApp.scala +++ b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/ScanApp.scala @@ -254,24 +254,26 @@ class ScanApp( ) kvStore <- ScanKeyValueStore(dsoParty, participantId, storage, loggerFactory) kvProvider = new ScanKeyValueProvider(kvStore, loggerFactory) - bulkStorage = (config.bulkStorage.staging, config.bulkStorage.committed).tupled.map(_ => - BulkStorage( - scanStorageConfigV1, - config.bulkStorage, - acsSnapshotStore, - updateHistory, - currentMigrationId = domainMigrationId, - kvProvider, - retryProvider.metricsFactory, - config.automation, - backoffClock = new WallClock(retryProvider.timeouts, loggerFactory), - store, - svName, - ledgerClient, - amuletAppParameters.upgradesConfig, - retryProvider, - loggerFactory, - ) + bulkStorage <- (config.bulkStorage.staging, config.bulkStorage.committed).tupled.traverse(_ => + appInitStep("Initialize bulk storage") { + BulkStorage( + scanStorageConfigV1, + config.bulkStorage, + acsSnapshotStore, + updateHistory, + currentMigrationId = domainMigrationId, + kvProvider, + retryProvider.metricsFactory, + config.automation, + backoffClock = new WallClock(retryProvider.timeouts, loggerFactory), + store, + svName, + ledgerClient, + amuletAppParameters.upgradesConfig, + retryProvider, + loggerFactory, + ) + } ) appActivityRecordStore = new DbAppActivityRecordStore( storage, diff --git a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/BftScanConnection.scala b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/BftScanConnection.scala index ccd90902ec..30802acef6 100644 --- a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/BftScanConnection.scala +++ b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/BftScanConnection.scala @@ -16,7 +16,6 @@ import org.lfdecentralizedtrust.splice.codegen.java.splice.amuletrules.{ AmuletRules, TransferPreapproval, } -import org.lfdecentralizedtrust.splice.codegen.java.splice.dsorules.DsoRules import org.lfdecentralizedtrust.splice.codegen.java.splice.externalpartyamuletrules.{ ExternalPartyAmuletRules, TransferCommandCounter, @@ -38,7 +37,6 @@ import org.lfdecentralizedtrust.splice.http.HttpClient import org.lfdecentralizedtrust.splice.http.v0.definitions.{ AnsEntry, GetBulkObjectChecksumsResponse, - GetDsoInfoResponse, GetRewardAccountingActivityTotalsResponse, GetRewardAccountingBatchResponse, GetRewardAccountingRootHashResponse, @@ -70,6 +68,7 @@ import org.lfdecentralizedtrust.splice.util.{ ChoiceContextWithDisclosures, Contract, ContractWithState, + DsoInfo, FactoryChoiceWithDisclosures, TemplateJsonDecoder, } @@ -106,6 +105,7 @@ import org.lfdecentralizedtrust.splice.codegen.java.splice.api.token.allocationi import org.lfdecentralizedtrust.splice.codegen.java.splice.api.token.transferinstructionv1 import org.lfdecentralizedtrust.splice.codegen.java.splice.api.token.transferinstructionv2 import org.lfdecentralizedtrust.splice.codegen.java.splice.dsorules.{ + DsoRules, DsoRules_CloseVoteRequestResult, VoteRequest, } @@ -195,12 +195,21 @@ class BftScanConnection( override def getDsoInfo()(implicit ec: ExecutionContext, tc: TraceContext, - ): Future[GetDsoInfoResponse] = + ): Future[DsoInfo] = bftCall( _.getDsoInfo(), "getDsoInfo", ) + override def getDsoRules( + )(implicit + tc: TraceContext + ): Future[Contract[DsoRules.ContractId, DsoRules]] = + bftCall( + _.getDsoInfo().map(_.dsoRules.contract), + "getDsoRules", + ) + override def getHoldingsSummaryAt( at: CantonTimestamp, migrationId: Long, @@ -234,12 +243,6 @@ class BftScanConnection( "getAmuletRulesWithState", ) - override def getDsoRules( - )(implicit - tc: TraceContext - ): Future[Contract[DsoRules.ContractId, DsoRules]] = - bftCall(_.getDsoRules(), "getDsoRules") - override protected def runGetExternalPartyAmuletRules( cachedExternalPartyAmuletRules: Option[ ContractWithState[ExternalPartyAmuletRules.ContractId, ExternalPartyAmuletRules] @@ -902,13 +905,30 @@ class BftScanConnection( endpoint: String, callConfig: BftCallConfig = BftCallConfig.default(scanList.scanConnections), consensusFailureLogLevel: Level = Level.WARN, - consensusLogConfig: BftScanConnection.ConsensusLogConfig = - BftScanConnection.ConsensusLogConfig(), shortenResponsesForLog: T => Any = identity[T], )(implicit ec: ExecutionContext, tc: TraceContext, - ): Future[T] = { + ): Future[T] = bftCallWithScanUris( + call, + endpoint, + callConfig, + consensusFailureLogLevel, + shortenResponsesForLog = shortenResponsesForLog, + ) + .map(_._1) + + private def bftCallWithScanUris[T]( + call: SingleScanConnection => Future[T], + endpoint: String, + callConfig: BftCallConfig, + consensusFailureLogLevel: Level = Level.WARN, + disagreementLogLevel: Level = Level.INFO, + shortenResponsesForLog: T => Any = identity[T], + )(implicit + ec: ExecutionContext, + tc: TraceContext, + ): Future[(T, List[Uri])] = { implicit val mc: MetricsContext = MetricsContext("request" -> endpoint) val connections = scanList.scanConnections @@ -946,7 +966,7 @@ class BftScanConnection( nTargetSuccess = callConfig.targetSuccess, logger, shortenResponsesForLog, - consensusLogConfig, + disagreementLogLevel, connectionMetrics, ), logger, @@ -1011,15 +1031,21 @@ class BftScanConnection( override def getRewardAccountingActivityTotals(roundNumber: Long)(implicit ec: ExecutionContext, tc: TraceContext, - ): Future[GetRewardAccountingActivityTotalsResponse] = { + ): Future[GetRewardAccountingActivityTotalsResponse] = + getRewardAccountingActivityTotalsWithScanUris(roundNumber).map(_._1) + + def getRewardAccountingActivityTotalsWithScanUris(roundNumber: Long)(implicit + ec: ExecutionContext, + tc: TraceContext, + ): Future[(GetRewardAccountingActivityTotalsResponse, List[Uri])] = { val undetermined = GetRewardAccountingActivityTotalsResponse( RewardAccountingActivityTotalsUndetermined(status = "Undetermined") ) val callConfig = BftCallConfig.default(scanList.scanConnections) - if (!callConfig.enoughAvailableScans) Future.successful(undetermined) + if (!callConfig.enoughAvailableScans) Future.successful((undetermined, Nil)) else - bftCall[RewardAccountingActivityTotalsOk]( + bftCallWithScanUris[RewardAccountingActivityTotalsOk]( call = scan => scan.getRewardAccountingActivityTotals(roundNumber).flatMap { case GetRewardAccountingActivityTotalsResponse.members @@ -1031,19 +1057,13 @@ class BftScanConnection( }, endpoint = "getRewardAccountingActivityTotals", callConfig = callConfig, - consensusLogConfig = BftScanConnection.ConsensusLogConfig( - disagreementLogLevel = Level.WARN, - onlyLogDisagreementsInSuccessResponse = true, - agreementLogLevel = Some(Level.INFO), - ), + disagreementLogLevel = Level.WARN, ) - .transform(tryTotals => - Success( - tryTotals.toOption.fold(undetermined)(ok => - GetRewardAccountingActivityTotalsResponse(ok) - ) - ) - ) + .transformWith { + case Success((totals, consensusUris)) => + Future.successful((GetRewardAccountingActivityTotalsResponse(totals), consensusUris)) + case Failure(_) => Future.successful((undetermined, Nil)) + } } /** This is special because in addition to 'Ok' we can receive @@ -1059,15 +1079,21 @@ class BftScanConnection( override def getRewardAccountingRootHash(roundNumber: Long)(implicit ec: ExecutionContext, tc: TraceContext, - ): Future[GetRewardAccountingRootHashResponse] = { + ): Future[GetRewardAccountingRootHashResponse] = + getRewardAccountingRootHashWithScanUris(roundNumber).map(_._1) + + def getRewardAccountingRootHashWithScanUris(roundNumber: Long)(implicit + ec: ExecutionContext, + tc: TraceContext, + ): Future[(GetRewardAccountingRootHashResponse, List[Uri])] = { val undetermined = GetRewardAccountingRootHashResponse( RewardAccountingRootHashUndetermined(status = "Undetermined") ) val callConfig = BftCallConfig.default(scanList.scanConnections) - if (!callConfig.enoughAvailableScans) Future.successful(undetermined) + if (!callConfig.enoughAvailableScans) Future.successful((undetermined, Nil)) else - bftCall[String]( + bftCallWithScanUris[String]( call = scan => scan.getRewardAccountingRootHash(roundNumber).flatMap { case GetRewardAccountingRootHashResponse.members.RewardAccountingRootHashOk(ok) => @@ -1078,25 +1104,24 @@ class BftScanConnection( }, endpoint = "getRewardAccountingRootHash", callConfig = callConfig, - consensusLogConfig = BftScanConnection.ConsensusLogConfig( - disagreementLogLevel = Level.WARN, - onlyLogDisagreementsInSuccessResponse = true, - agreementLogLevel = Some(Level.INFO), - ), + disagreementLogLevel = Level.WARN, ) - .transform(tryRootHash => - Success( - tryRootHash.toOption.fold(undetermined)(rootHash => - GetRewardAccountingRootHashResponse( - RewardAccountingRootHashOk( - status = "Ok", - roundNumber = roundNumber, - rootHash = rootHash, - ) + .transformWith { + case Success((rootHash, consensusUris)) => + Future.successful( + ( + GetRewardAccountingRootHashResponse( + RewardAccountingRootHashOk( + status = "Ok", + roundNumber = roundNumber, + rootHash = rootHash, + ) + ), + consensusUris.map(_.toString), ) ) - ) - ) + case Failure(_) => Future.successful((undetermined, Nil)) + } } /** The batch contents are verifiable via the hash, so BFT agreement across scans is not @@ -1142,19 +1167,19 @@ object BftScanConnection { nTargetSuccess: Int, logger: TracedLogger, shortenResponsesForLog: T => Any = identity[T], - consensusLogConfig: ConsensusLogConfig = ConsensusLogConfig(), + disagreementLogLevel: Level = Level.INFO, connectionMetrics: Option[ScanConnectionMetrics] = None, )(implicit ec: ExecutionContext, tc: TraceContext, mc: MetricsContext = MetricsContext.Empty, - ): Future[T] = { + ): Future[(T, List[Uri])] = { require(requestFrom.nonEmpty, "At least one request must be made.") val responses = new ConcurrentHashMap[BftScanConnection.ScanResponse[T], List[Uri]]() val nResponsesDone = new AtomicInteger(0) - val finalResponse = Promise[T]() + val finalResponse = Promise[(T, List[Uri])]() requestFrom.foreach { scan => call(scan) @@ -1173,7 +1198,7 @@ object BftScanConnection { case _ => true } if (considerResponseForQuorum && agreements.size == nTargetSuccess) { // consensus has been reached - finalResponse.tryComplete(response): Unit + finalResponse.tryComplete(response.map(r => (r, agreements))): Unit } if (nResponsesDone.incrementAndGet() == requestFrom.size) { // all Scans are done @@ -1188,9 +1213,9 @@ object BftScanConnection { case Some(consensusResponse) => logDisagreements( logger, - consensusResponse, + consensusResponse.map(_._1), responses, - consensusLogConfig, + disagreementLogLevel, connectionMetrics, ) } @@ -1236,7 +1261,7 @@ object BftScanConnection { logger: TracedLogger, consensusResponse: Try[T], responses: ConcurrentHashMap[BftScanConnection.ScanResponse[T], List[Uri]], - consensusLogConfig: ConsensusLogConfig, + disagreementLogLevel: Level, connectionMetrics: Option[ScanConnectionMetrics], )(implicit ec: ExecutionContext, tc: TraceContext, mc: MetricsContext): Unit = { implicit val elc: ErrorLoggingContext = ErrorLoggingContext.fromTracedLogger(logger) @@ -1266,28 +1291,16 @@ object BftScanConnection { keyToGroupResponses(consensusResponse).foreach { consensusResponseKey => val agreeingScanUrls = responses.remove(consensusResponseKey) agreeingScanUrls.foreach(recordConsensus(_, "agree", Map.empty)) - consensusLogConfig.agreementLogLevel.foreach { level => - LoggerUtil.logAtLevel( - level, - s"Reached consensus from:\n${agreeingScanUrls.mkString("\n")}", - ) - } responses.forEach { (disagreeingResponse, scanUrls) => val extraLabels = disagreementLabels(disagreeingResponse) scanUrls.foreach(recordConsensus(_, "disagree", extraLabels)) - val shouldLog = disagreeingResponse match { - case _: SuccessfulResponse[?] => true - case _ => !consensusLogConfig.onlyLogDisagreementsInSuccessResponse - } - if (shouldLog) { - LoggerUtil.logAtLevel( - consensusLogConfig.disagreementLogLevel, - s"""The following Scan URLs disagreed with consensus: - |${scanUrls.map(url => s" $url").mkString("\n")} - |consensus response: $consensusResponse - |disagreeing response: $disagreeingResponse""".stripMargin, - ) - } + LoggerUtil.logAtLevel( + disagreementLogLevel, + s"""The following Scan URLs disagreed with consensus: + |${scanUrls.map(url => s" $url").mkString("\n")} + |consensus response: $consensusResponse + |disagreeing response: $disagreeingResponse""".stripMargin, + ) } } } @@ -2159,12 +2172,6 @@ object BftScanConnection { extends RuntimeException(s"Scan $url has no answer to contribute to consensus") with NoStackTrace - case class ConsensusLogConfig( - disagreementLogLevel: Level = Level.INFO, - onlyLogDisagreementsInSuccessResponse: Boolean = false, - agreementLogLevel: Option[Level] = None, - ) - private sealed trait ScanResponse[+T] private case class SuccessfulResponse[+T](response: T) extends ScanResponse[T] private case class HttpFailureResponse[+T](status: StatusCode, body: Json) extends ScanResponse[T] diff --git a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/ScanConnection.scala b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/ScanConnection.scala index bbf758cb1c..1aff1a2ffc 100644 --- a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/ScanConnection.scala +++ b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/ScanConnection.scala @@ -25,7 +25,6 @@ import org.lfdecentralizedtrust.splice.environment.* import org.lfdecentralizedtrust.splice.http.HttpClient import org.lfdecentralizedtrust.splice.http.v0.definitions.{ GetBulkObjectChecksumsResponse, - GetDsoInfoResponse, GetRewardAccountingActivityTotalsResponse, GetRewardAccountingBatchResponse, GetRewardAccountingRootHashResponse, @@ -74,7 +73,7 @@ trait ScanConnection def getDsoPartyId()(implicit ec: ExecutionContext, tc: TraceContext): Future[PartyId] - def getDsoInfo()(implicit ec: ExecutionContext, tc: TraceContext): Future[GetDsoInfoResponse] + def getDsoInfo()(implicit ec: ExecutionContext, tc: TraceContext): Future[DsoInfo] /** Query for the DSO party id, retrying until it succeeds. * @@ -109,9 +108,7 @@ trait ScanConnection tc: TraceContext, ): Future[ContractWithState[AmuletRules.ContractId, AmuletRules]] - def getDsoRules()(implicit - tc: TraceContext - ): Future[Contract[DsoRules.ContractId, DsoRules]] + def getDsoRules()(implicit tc: TraceContext): Future[Contract[DsoRules.ContractId, DsoRules]] def getExternalPartyAmuletRules()(implicit ec: ExecutionContext, diff --git a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/SingleScanConnection.scala b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/SingleScanConnection.scala index e0852b971a..a0af1d2526 100644 --- a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/SingleScanConnection.scala +++ b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/SingleScanConnection.scala @@ -4,7 +4,6 @@ package org.lfdecentralizedtrust.splice.scan.admin.api.client import cats.data.OptionT -import cats.syntax.either.* import com.daml.metrics.api.MetricsContext import org.lfdecentralizedtrust.splice.codegen.java.splice.amulet.{ FeaturedAppRight, @@ -26,6 +25,7 @@ import org.lfdecentralizedtrust.splice.codegen.java.splice.round.{ import org.lfdecentralizedtrust.splice.codegen.java.splice.ans.AnsRules import org.lfdecentralizedtrust.splice.config.UpgradesConfig import org.lfdecentralizedtrust.splice.environment.{ + BaseAppConnection, HttpAppConnection, RetryProvider, SpliceLedgerClient, @@ -51,6 +51,7 @@ import org.lfdecentralizedtrust.splice.util.{ ChoiceContextWithDisclosures, Contract, ContractWithState, + DsoInfo, FactoryChoiceWithDisclosures, TemplateJsonDecoder, } @@ -78,9 +79,8 @@ import org.lfdecentralizedtrust.splice.codegen.java.splice.dsorules.{ DsoRules_CloseVoteRequestResult, VoteRequest, } -import io.grpc.Status import org.apache.pekko.http.scaladsl.model.{HttpHeader, Uri} -import org.lfdecentralizedtrust.splice.admin.api.client.commands.HttpCommand +import org.lfdecentralizedtrust.splice.admin.api.client.commands.{HttpCommand, HttpCommandException} import org.lfdecentralizedtrust.splice.codegen.java.splice.api.token.transferinstructionv1 import org.lfdecentralizedtrust.splice.codegen.java.splice.api.token.transferinstructionv2 import org.lfdecentralizedtrust.splice.codegen.java.splice.api.token.allocationv1 @@ -142,9 +142,11 @@ class SingleScanConnection private[client] ( .runHttpCmd(url, command, headers) .andThen { case Failure(e) => - MetricsContext.withMetricLabels(("outcome", e.getClass.getSimpleName)) { - implicit ec2 => - metrics.callPerConnection.mark()(m.merge(ec2)) + MetricsContext.withMetricLabels( + ("outcome", e.getClass.getSimpleName), + ("http_status", SingleScanConnection.httpStatusLabel(e)), + ) { implicit ec2 => + metrics.callPerConnection.mark()(m.merge(ec2)) } timer.stop()(m) case Success(_) => @@ -184,7 +186,7 @@ class SingleScanConnection private[client] ( override def getDsoInfo()(implicit ec: ExecutionContext, tc: TraceContext, - ): Future[org.lfdecentralizedtrust.splice.http.v0.definitions.GetDsoInfoResponse] = { + ): Future[DsoInfo] = { runHttpCmd(config.adminApi.url, HttpScanAppClient.GetDsoInfo(List())) } @@ -247,18 +249,7 @@ class SingleScanConnection private[client] ( )(implicit tc: TraceContext ): Future[Contract[DsoRules.ContractId, DsoRules]] = { - runHttpCmd( - config.adminApi.url, - HttpScanAppClient.GetDsoInfo(headers = List()), - ).map { dsoInfo => - Contract - .fromHttp(DsoRules.COMPANION)(dsoInfo.dsoRules.contract) - .valueOr(err => - throw Status.INVALID_ARGUMENT - .withDescription(s"Failed to decode dso rules: $err") - .asRuntimeException - ) - } + getDsoInfo().map(_.dsoRules.contract) } override def listVoteRequests()(implicit @@ -1039,6 +1030,18 @@ class SingleScanConnection private[client] ( } object SingleScanConnection { + + private[client] def httpStatusLabel(error: Throwable): String = + error match { + case e: BaseAppConnection.UnexpectedHttpJsonResponse => e.statusCode.intValue.toString + case e: BaseAppConnection.UnexpectedHttpMalformedJsonResponse => + e.statusCode.intValue.toString + case e: BaseAppConnection.UnexpectedHttpTextResponse => e.statusCode.intValue.toString + case e: BaseAppConnection.UnexpectedHttpNonJsonResponse => e.statusCode.intValue.toString + case e: HttpCommandException => e.status.intValue.toString + case _ => "none" + } + def withSingleScanConnection[T]( scanConfig: ScanAppClientConfig, upgradesConfig: UpgradesConfig, diff --git a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/commands/HttpScanAppClient.scala b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/commands/HttpScanAppClient.scala index 11f3149683..eb15cf4536 100644 --- a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/commands/HttpScanAppClient.scala +++ b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/commands/HttpScanAppClient.scala @@ -62,6 +62,7 @@ import org.lfdecentralizedtrust.splice.util.{ Contract, ContractWithState, DomainRecordTimeRange, + DsoInfo, FactoryChoiceWithDisclosures, PackageQualifiedName, TemplateJsonDecoder, @@ -203,7 +204,7 @@ object HttpScanAppClient { } case class GetDsoInfo(headers: List[HttpHeader]) - extends InternalBaseCommand[http.GetDsoInfoResponse, definitions.GetDsoInfoResponse] { + extends InternalBaseCommand[http.GetDsoInfoResponse, DsoInfo] { override def submitRequest( client: ScanClient, @@ -214,7 +215,7 @@ object HttpScanAppClient { override def handleOk()(implicit decoder: TemplateJsonDecoder ) = { case http.GetDsoInfoResponse.OK(response) => - Right(response) + DsoInfo.fromHttp(response) } } @@ -1071,6 +1072,48 @@ object HttpScanAppClient { } } + case class GetAcsSnapshotAtV2( + at: java.time.OffsetDateTime, + migrationId: Long, + recordTimeMatch: Option[definitions.AcsRequestV2.RecordTimeMatch], + after: Option[String] = None, + pageSize: Int = 100, + partyIds: Option[Vector[PartyId]] = None, + templates: Option[Vector[PackageQualifiedName]] = None, + ) extends InternalBaseCommand[ + http.GetAcsSnapshotAtV2Response, + Option[definitions.AcsResponseV2], + ] { + override def submitRequest( + client: ScanClient, + headers: List[HttpHeader], + ): EitherT[Future, Either[Throwable, HttpResponse], http.GetAcsSnapshotAtV2Response] = + client.getAcsSnapshotAtV2( + definitions.AcsRequestV2( + migrationId, + at, + recordTimeMatch, + after, + pageSize, + partyIds.map(_.map(_.toProtoPrimitive)), + templates.map(_.map(_.toString)), + ), + headers, + ) + + override protected def handleOk()(implicit + decoder: TemplateJsonDecoder + ): PartialFunction[http.GetAcsSnapshotAtV2Response, Either[ + String, + Option[definitions.AcsResponseV2], + ]] = { + case http.GetAcsSnapshotAtV2Response.OK(value) => + Right(Some(value)) + case http.GetAcsSnapshotAtV2Response.NotFound(_) => + Right(None) + } + } + case class GetHoldingsStateAt( at: java.time.OffsetDateTime, migrationId: Long, diff --git a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/admin/http/HttpScanHandler.scala b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/admin/http/HttpScanHandler.scala index f50a470cd2..1950aead98 100644 --- a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/admin/http/HttpScanHandler.scala +++ b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/admin/http/HttpScanHandler.scala @@ -67,6 +67,7 @@ import org.lfdecentralizedtrust.splice.http.{ import org.lfdecentralizedtrust.splice.http.v0.{definitions, scan as v0} import org.lfdecentralizedtrust.splice.http.v0.definitions.{ AcsRequest, + AcsRequestV2, BatchListVotesByVoteRequestsRequest, CountVoteResultsRequest, DamlValueEncoding, @@ -74,6 +75,7 @@ import org.lfdecentralizedtrust.splice.http.v0.definitions.{ EventHistoryRequest, GetBulkObjectChecksumsRequest, HoldingsStateRequest, + HoldingsStateRequestV2, HoldingsSummaryRequest, HoldingsSummaryRequestV1, ListBulkUpdateHistoryObjectsRequest, @@ -98,8 +100,12 @@ import org.lfdecentralizedtrust.splice.scan.store.{ ScanStore, TxLogEntry, } +import org.lfdecentralizedtrust.splice.scan.store.AppActivityStore.RoundIngestionStatus import org.lfdecentralizedtrust.splice.scan.store.bulk.BulkStorageReader -import org.lfdecentralizedtrust.splice.scan.store.AcsSnapshotStore.QueryAcsSnapshotResult +import org.lfdecentralizedtrust.splice.scan.store.AcsSnapshotStore.{ + QueryAcsSnapshotPaginationToken, + QueryAcsSnapshotResult, +} import org.lfdecentralizedtrust.splice.scan.store.bulk.AcsSnapshotBulkStorage.AcsSnapshotObjects import org.lfdecentralizedtrust.splice.scan.store.bulk.UpdateHistoryBulkStorage.UpdateHistoryObjectsResponse import org.lfdecentralizedtrust.splice.store.AppStoreWithIngestion.SpliceLedgerConnectionPriority @@ -108,6 +114,7 @@ import org.lfdecentralizedtrust.splice.store.{ AppStore, AppStoreWithIngestion, PageLimit, + TimestampWithMigrationId, VoteResultsFilters, VotesStore, } @@ -122,6 +129,7 @@ import org.lfdecentralizedtrust.splice.util.{ Codec, Contract, ContractWithState, + DsoInfo, PackageQualifiedName, QualifiedName, } @@ -210,17 +218,17 @@ class HttpScanHandler( amuletRules <- store.getAmuletRulesWithState() rulesAndStates <- store.getDsoRulesWithStateWithSvNodeStates() dsoRules = rulesAndStates.dsoRules - } yield definitions.GetDsoInfoResponse( + } yield DsoInfo( svUser = svUserName, - svPartyId = svParty.toProtoPrimitive, - dsoPartyId = store.key.dsoParty.toProtoPrimitive, + svParty = svParty, + dsoParty = store.key.dsoParty, votingThreshold = Thresholds.requiredNumVotes(dsoRules), - latestMiningRound = latestOpenMiningRound.toContractWithState.toHttp, - amuletRules = amuletRules.toHttp, - dsoRules = dsoRules.toHttp, - svNodeStates = rulesAndStates.svNodeStates.values.map(_.toHttp).toVector, + latestMiningRound = latestOpenMiningRound.toContractWithState, + amuletRules = amuletRules, + dsoRules = dsoRules, + svNodeStates = rulesAndStates.svNodeStates, initialRound = Some(initialRound), - ) + ).toHttp } } @@ -704,9 +712,9 @@ class HttpScanHandler( implicit val tc: TraceContext = extracted val afterO = after.map { after => val afterRecordTime = parseTimestamp(after.afterRecordTime) - ( - after.afterMigrationId, + TimestampWithMigrationId( afterRecordTime, + after.afterMigrationId, ) } confirmBackfillingIsCompleteThen(updateHistory) { @@ -896,7 +904,7 @@ class HttpScanHandler( implicit val tc: TraceContext = extracted val afterO = after.map { a => val afterRecordTime = parseTimestamp(a.afterRecordTime) - (a.afterMigrationId, afterRecordTime) + TimestampWithMigrationId(afterRecordTime, a.afterMigrationId) } confirmBackfillingIsCompleteThen(updateHistory) { @@ -1504,19 +1512,18 @@ class HttpScanHandler( } // Shared between /v0/state/acs and /v1/state/acs. The only difference between them is in `toResponse`. - private def acsSnapshotQuery[T](request: AcsRequest, toResponse: QueryAcsSnapshotResult => T)( - implicit tc: TraceContext + private def acsSnapshotQuery[T]( + migrationId: Long, + recordTime: java.time.OffsetDateTime, + recordTimeIsAtOrBefore: Boolean, + after: Option[AcsSnapshotStore.QueryAcsSnapshotPaginationToken], + pageSize: Int, + partyIds: Option[Vector[String]], + templates: Option[Vector[String]], + toResponse: QueryAcsSnapshotResult => T, + )(implicit + tc: TraceContext ): Future[Either[String, T]] = { - val AcsRequest( - migrationId, - recordTime, - recordTimeMatch, - after, - pageSize, - partyIds, - templates, - ) = request - def exactQuery(recordTimeTs: CantonTimestamp) = snapshotStore .queryAcsSnapshot( migrationId, @@ -1541,7 +1548,7 @@ class HttpScanHandler( queryWithOptionalAtOrBefore( migrationId, recordTime, - recordTimeMatch.contains(AcsRequest.RecordTimeMatch.AtOrBefore), + recordTimeIsAtOrBefore, exactQuery, toResponse, ) @@ -1561,7 +1568,9 @@ class HttpScanHandler( event.event, ) ), - result.afterToken, + result.afterToken.map { + case QueryAcsSnapshotPaginationToken.RowIdQueryAcsSnapshotPaginationToken(after) => after + }, ) } @@ -1579,7 +1588,26 @@ class HttpScanHandler( event.event, ) ), - result.afterToken, + result.afterToken.map { + case QueryAcsSnapshotPaginationToken.RowIdQueryAcsSnapshotPaginationToken(after) => after + }, + ) + + private def toAcsV2Response(migrationId: Long, result: QueryAcsSnapshotResult)(implicit + tc: TraceContext + ) = + definitions.AcsResponseV2( + Codec.encode(result.snapshotRecordTime), + migrationId, + result.createdEventsInPage + .map(event => + CompactJsonScanHttpEncodings().javaToHttpActiveContract( + event.eventId, + event.recordTime, + event.event, + ) + ), + result.afterToken.map(_.encodeToBase64), ) override def getAcsSnapshotAt(respond: ScanResource.GetAcsSnapshotAtResponse.type)( @@ -1593,7 +1621,19 @@ class HttpScanHandler( ) withSpan(s"$workflowId.getAcsSnapshotAt") { _ => _ => - acsSnapshotQuery(body, toResponse).map { + acsSnapshotQuery( + migrationId = body.migrationId, + recordTime = body.recordTime, + recordTimeIsAtOrBefore = + body.recordTimeMatch.contains(AcsRequest.RecordTimeMatch.AtOrBefore), + after = body.after.map( + AcsSnapshotStore.QueryAcsSnapshotPaginationToken.RowIdQueryAcsSnapshotPaginationToken(_) + ), + pageSize = body.pageSize, + partyIds = body.partyIds, + templates = body.templates, + toResponse = toResponse, + ).map { case Right(response) => response case Left(errorMessage) => ScanResource.GetAcsSnapshotAtResponseNotFound( @@ -1615,7 +1655,19 @@ class HttpScanHandler( } withSpan(s"$workflowId.getAcsSnapshotAtV1") { _ => _ => - acsSnapshotQuery(body, toResponse).map { + acsSnapshotQuery( + migrationId = body.migrationId, + recordTime = body.recordTime, + recordTimeIsAtOrBefore = + body.recordTimeMatch.contains(AcsRequest.RecordTimeMatch.AtOrBefore), + after = body.after.map( + AcsSnapshotStore.QueryAcsSnapshotPaginationToken.RowIdQueryAcsSnapshotPaginationToken(_) + ), + pageSize = body.pageSize, + partyIds = body.partyIds, + templates = body.templates, + toResponse = toResponse, + ).map { case Right(response) => response case Left(errorMessage) => ScanResource.GetAcsSnapshotAtV1ResponseNotFound( @@ -1625,21 +1677,50 @@ class HttpScanHandler( } } + override def getAcsSnapshotAtV2(respond: ScanResource.GetAcsSnapshotAtV2Response.type)( + body: AcsRequestV2 + )(extracted: TraceContext): Future[ScanResource.GetAcsSnapshotAtV2Response] = { + implicit val tc: TraceContext = extracted + + def toResponse(result: QueryAcsSnapshotResult) = { + ScanResource.GetAcsSnapshotAtV2ResponseOK( + toAcsV2Response(body.migrationId, result) + ) + } + + withSpan(s"$workflowId.getAcsSnapshotAtV1") { _ => _ => + acsSnapshotQuery( + migrationId = body.migrationId, + recordTime = body.recordTime, + recordTimeIsAtOrBefore = + body.recordTimeMatch.contains(AcsRequestV2.RecordTimeMatch.AtOrBefore), + after = + body.after.map(AcsSnapshotStore.QueryAcsSnapshotPaginationToken.tryDecodeFromBase64), + pageSize = body.pageSize, + partyIds = body.partyIds, + templates = body.templates, + toResponse = toResponse, + ).map { + case Right(response) => response + case Left(errorMessage) => + ScanResource.GetAcsSnapshotAtV2ResponseNotFound( + ErrorResponse(errorMessage) + ) + } + } + } + private def holdingStateQuery[T]( - request: HoldingsStateRequest, + migrationId: Long, + recordTime: java.time.OffsetDateTime, + recordTimeIsAtOrBefore: Boolean, + after: Option[AcsSnapshotStore.QueryAcsSnapshotPaginationToken], + pageSize: Int, + ownerPartyIds: Vector[String], toResponse: QueryAcsSnapshotResult => T, )(implicit tc: TraceContext ): Future[Either[String, T]] = { - val HoldingsStateRequest( - migrationId, - recordTime, - recordTimeMatch, - after, - pageSize, - ownerPartyIds, - ) = request - def exactQuery(recordTimeTs: CantonTimestamp) = snapshotStore .getHoldingsState( migrationId, @@ -1652,7 +1733,7 @@ class HttpScanHandler( queryWithOptionalAtOrBefore( migrationId, recordTime, - recordTimeMatch.contains(HoldingsStateRequest.RecordTimeMatch.AtOrBefore), + recordTimeIsAtOrBefore, exactQuery, toResponse, ) @@ -1666,7 +1747,18 @@ class HttpScanHandler( ScanResource.GetHoldingsStateAtResponseOK(toAcsV0Response(body.migrationId, result)) withSpan(s"$workflowId.getHoldingsStateAt") { _ => _ => - holdingStateQuery(body, toResponse).map { + holdingStateQuery( + migrationId = body.migrationId, + recordTime = body.recordTime, + recordTimeIsAtOrBefore = + body.recordTimeMatch.contains(HoldingsStateRequest.RecordTimeMatch.AtOrBefore), + after = body.after.map( + AcsSnapshotStore.QueryAcsSnapshotPaginationToken.RowIdQueryAcsSnapshotPaginationToken(_) + ), + pageSize = body.pageSize, + ownerPartyIds = body.ownerPartyIds, + toResponse = toResponse, + ).map { case Right(response) => response case Left(errorMessage) => ScanResource.GetHoldingsStateAtResponseNotFound( @@ -1684,7 +1776,18 @@ class HttpScanHandler( ScanResource.GetHoldingsStateAtV1ResponseOK(toAcsV1Response(body.migrationId, result)) withSpan(s"$workflowId.getHoldingsStateAtV1") { _ => _ => - holdingStateQuery(body, toResponse).map { + holdingStateQuery( + migrationId = body.migrationId, + recordTime = body.recordTime, + recordTimeIsAtOrBefore = + body.recordTimeMatch.contains(HoldingsStateRequest.RecordTimeMatch.AtOrBefore), + after = body.after.map( + AcsSnapshotStore.QueryAcsSnapshotPaginationToken.RowIdQueryAcsSnapshotPaginationToken(_) + ), + pageSize = body.pageSize, + ownerPartyIds = body.ownerPartyIds, + toResponse, + ).map { case Right(response) => response case Left(errorMessage) => ScanResource.GetHoldingsStateAtV1ResponseNotFound( @@ -1694,6 +1797,34 @@ class HttpScanHandler( } } + override def getHoldingsStateAtV2(respond: ScanResource.GetHoldingsStateAtV2Response.type)( + body: HoldingsStateRequestV2 + )(extracted: TraceContext): Future[ScanResource.GetHoldingsStateAtV2Response] = { + implicit val tc: TraceContext = extracted + def toResponse(result: QueryAcsSnapshotResult) = + ScanResource.GetHoldingsStateAtV2ResponseOK(toAcsV2Response(body.migrationId, result)) + + withSpan(s"$workflowId.getHoldingsStateAtV1") { _ => _ => + holdingStateQuery( + migrationId = body.migrationId, + recordTime = body.recordTime, + recordTimeIsAtOrBefore = + body.recordTimeMatch.contains(HoldingsStateRequestV2.RecordTimeMatch.AtOrBefore), + after = + body.after.map(AcsSnapshotStore.QueryAcsSnapshotPaginationToken.tryDecodeFromBase64), + pageSize = body.pageSize, + ownerPartyIds = body.ownerPartyIds, + toResponse, + ).map { + case Right(response) => response + case Left(errorMessage) => + ScanResource.GetHoldingsStateAtV2ResponseNotFound( + ErrorResponse(errorMessage) + ) + } + } + } + override def getHoldingsSummaryAt(respond: ScanResource.GetHoldingsSummaryAtResponse.type)( body: HoldingsSummaryRequest )(extracted: TraceContext): Future[ScanResource.GetHoldingsSummaryAtResponse] = { @@ -2774,11 +2905,9 @@ class HttpScanHandler( undetermined } case None => - appActivityStore.earliestIngestedRound().map { - case Some(earliestIngested) if roundNumber <= earliestIngested => - cannotProvide - case _ => - undetermined + appActivityStore.ingestionStatusForRound(roundNumber).map { + case RoundIngestionStatus.CannotProvide => cannotProvide + case RoundIngestionStatus.Undetermined => undetermined } } } @@ -2815,11 +2944,9 @@ class HttpScanHandler( ) ) case None => - appActivityStore.earliestIngestedRound().map { - case Some(earliestIngested) if roundNumber <= earliestIngested => - cannotProvide - case _ => - undetermined + appActivityStore.ingestionStatusForRound(roundNumber).map { + case RoundIngestionStatus.CannotProvide => cannotProvide + case RoundIngestionStatus.Undetermined => undetermined } } } diff --git a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/automation/ScanHistoryBackfillingTrigger.scala b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/automation/ScanHistoryBackfillingTrigger.scala index a10390084f..22f9c15175 100644 --- a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/automation/ScanHistoryBackfillingTrigger.scala +++ b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/automation/ScanHistoryBackfillingTrigger.scala @@ -26,6 +26,7 @@ import org.lfdecentralizedtrust.splice.store.{ HistoryMetrics, ImportUpdatesBackfilling, PageLimit, + TimestampWithMigrationId, TreeUpdateWithMigrationId, UpdateHistory, } @@ -85,7 +86,7 @@ class ScanHistoryBackfillingTrigger( */ @SuppressWarnings(Array("org.wartremover.warts.Var")) @volatile - private var findHistoryStartAfter: Option[(Long, CantonTimestamp)] = None + private var findHistoryStartAfter: Option[TimestampWithMigrationId] = None @SuppressWarnings(Array("org.wartremover.warts.Var")) @volatile @@ -214,7 +215,8 @@ class ScanHistoryBackfillingTrigger( PageLimit.tryCreate(batchSize), ) _ = updates.lastOption.foreach(u => - findHistoryStartAfter = Some(u.migrationId -> u.update.update.recordTime) + findHistoryStartAfter = + Some(TimestampWithMigrationId(u.update.update.recordTime, u.migrationId)) ) result <- if (updates.isEmpty) { @@ -311,7 +313,7 @@ class ScanHistoryBackfillingTrigger( object ScanHistoryBackfillingTrigger { sealed trait Task extends PrettyPrinting final case class InitializeBackfillingTask( - after: Option[(Long, CantonTimestamp)] + after: Option[TimestampWithMigrationId] ) extends Task { override def pretty: Pretty[this.type] = prettyOfClass(param("after", _.after)) diff --git a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/config/ScanAppConfig.scala b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/config/ScanAppConfig.scala index 9273e016ed..423efc8d51 100644 --- a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/config/ScanAppConfig.scala +++ b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/config/ScanAppConfig.scala @@ -48,6 +48,21 @@ final case class BulkStorageConfig( staging: Option[S3Config] = None, committed: Option[S3Config] = None, bftCheckEnabled: Boolean = true, + /** When enabled, the app will reset all progress markers thus force recomputing data from genesis. + * Note that this does not delete any existing data, you usually would want to do that before setting + * this flag. Also, after restarting the app once with this flag enabled, you'd want to disable it back + * to avoid having the markers reset on every restart. + * TODO(#6251): this makes sense for initial stages of testing&deploying bulk storage, in case of + * encountered issues, but will not make sense when we start pruning the data from scan. We should remove + * this before starting to prune data. + */ + debugForceStartFromGenesis: Boolean = false, + /** A list of S3 object keys that this instance should not save to the committed bucket, and instead only + * delete from staging. To be used only in extreme cases where we decide to accept a BFT disagreement, + * and have the (minority of) disagreeing instances simply skip the broken objects. + * Should typically be used in test environments only. + */ + debugObjectsToNotCommit: Seq[String] = Seq.empty, ) /** @param miningRoundsCacheTimeToLiveOverride Intended only for testing! diff --git a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/rewards/AppActivityComputation.scala b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/rewards/AppActivityComputation.scala index be99679eee..2181d3fcd6 100644 --- a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/rewards/AppActivityComputation.scala +++ b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/rewards/AppActivityComputation.scala @@ -11,6 +11,7 @@ import com.digitalasset.daml.lf.data.Numeric import com.digitalasset.daml.lf.data.{assertRight as damlRight} import org.lfdecentralizedtrust.splice.scan.store.ScanRewardsReferenceStore import org.lfdecentralizedtrust.splice.scan.store.db.{DbAppActivityRecordStore, DbScanVerdictStore} +import org.lfdecentralizedtrust.splice.store.TimestampWithMigrationId import java.math.RoundingMode import scala.collection.immutable.SortedMap @@ -55,7 +56,7 @@ class AppActivityComputation( )(implicit tc: TraceContext): Future[Option[Long]] = rewardsReferenceStore .lookupActiveOpenMiningRounds(Seq(asOf)) - .map(_.get(asOf).map { case (roundNumber, _) => roundNumber }) + .map(_.get(asOf).map { case TimestampWithMigrationId(_, roundNumber) => roundNumber }) /** Compute app activity records for a batch of verdicts. * @@ -104,7 +105,7 @@ class AppActivityComputation( Future.successful((summary, verdict, None)) case (summary, verdict, true) => roundInfoByTime.get(summary.sequencingTime) match { - case Some((roundNumber, roundOpensAt)) => + case Some(TimestampWithMigrationId(roundOpensAt, roundNumber)) => for { featuredAppWeights <- rewardsReferenceStore.lookupFeaturedAppPartiesAsOf( roundOpensAt diff --git a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/AcsSnapshotStore.scala b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/AcsSnapshotStore.scala index c51128cd09..81721f9393 100644 --- a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/AcsSnapshotStore.scala +++ b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/AcsSnapshotStore.scala @@ -10,6 +10,7 @@ import org.lfdecentralizedtrust.splice.scan.store.AcsSnapshotStore.{ AcsSnapshot, IncrementalAcsSnapshot, IncrementalAcsSnapshotTable, + QueryAcsSnapshotPaginationToken, QueryAcsSnapshotResult, amuletQualifiedName, lockedAmuletQualifiedName, @@ -19,8 +20,8 @@ import org.lfdecentralizedtrust.splice.store.{HardLimit, Limit, LimitHelpers, Up import org.lfdecentralizedtrust.splice.store.db.{ AcsJdbcTypes, AcsQueries, - AdvisoryLocks, AdvisoryLockIds, + AdvisoryLocks, } import org.lfdecentralizedtrust.splice.util.{Contract, HoldingsSummary, PackageQualifiedName} import com.digitalasset.canton.data.CantonTimestamp @@ -38,6 +39,7 @@ import slick.jdbc.canton.ActionBasedSQLInterpolation.Implicits.actionBasedSQLInt import slick.jdbc.canton.SQLActionBuilder import slick.jdbc.{GetResult, JdbcProfile} +import java.nio.charset.StandardCharsets import java.util.concurrent.Semaphore import scala.concurrent.{ExecutionContext, Future} @@ -241,7 +243,7 @@ class AcsSnapshotStore( def queryAcsSnapshot( migrationId: Long, snapshot: CantonTimestamp, - after: Option[Long], + after: Option[QueryAcsSnapshotPaginationToken], limit: Limit, partyIds: Seq[PartyId], templates: Seq[PackageQualifiedName], @@ -267,7 +269,11 @@ class AcsSnapshotStore( ) ) begin <- after match { - case Some(value) if value < snapshot.firstRowId || value > snapshot.lastRowId => + case Some( + AcsSnapshotStore.QueryAcsSnapshotPaginationToken.RowIdQueryAcsSnapshotPaginationToken( + value + ) + ) if value < snapshot.firstRowId || value > snapshot.lastRowId => Future.failed( io.grpc.Status.INVALID_ARGUMENT .withDescription( @@ -275,7 +281,12 @@ class AcsSnapshotStore( ) .asRuntimeException() ) - case Some(value) => Future.successful(value + 1) + case Some( + AcsSnapshotStore.QueryAcsSnapshotPaginationToken.RowIdQueryAcsSnapshotPaginationToken( + value + ) + ) => + Future.successful(value + 1) case None => Future.successful(snapshot.firstRowId) } end = snapshot.lastRowId @@ -345,7 +356,9 @@ class AcsSnapshotStore( migrationId = migrationId, snapshotRecordTime = snapshot.snapshotRecordTime, createdEventsInPage = eventsInPage, - afterToken = afterToken, + afterToken = afterToken.map( + AcsSnapshotStore.QueryAcsSnapshotPaginationToken.RowIdQueryAcsSnapshotPaginationToken(_) + ), ) } } @@ -353,7 +366,7 @@ class AcsSnapshotStore( def getHoldingsState( migrationId: Long, snapshot: CantonTimestamp, - after: Option[Long], + after: Option[QueryAcsSnapshotPaginationToken], limit: Limit, partyIds: NonEmptyVector[PartyId], )(implicit tc: TraceContext): Future[QueryAcsSnapshotResult] = { @@ -906,11 +919,50 @@ object AcsSnapshotStore { ) } + sealed trait QueryAcsSnapshotPaginationToken { + def encodeToBase64: String = { + val jsonString = QueryAcsSnapshotPaginationToken.codec(this).noSpaces + java.util.Base64.getEncoder.encodeToString(jsonString.getBytes(StandardCharsets.UTF_8)) + } + } + object QueryAcsSnapshotPaginationToken { + case class RowIdQueryAcsSnapshotPaginationToken(after: Long) + extends QueryAcsSnapshotPaginationToken + + private val codec: io.circe.Codec[QueryAcsSnapshotPaginationToken] = + io.circe.Codec + .from(io.circe.Decoder[Long], io.circe.Encoder[Long]) + .iemap[QueryAcsSnapshotPaginationToken]((token: Long) => + Right(RowIdQueryAcsSnapshotPaginationToken(token)) + ) { case RowIdQueryAcsSnapshotPaginationToken(after) => after } + + def tryDecodeFromBase64(token: String): QueryAcsSnapshotPaginationToken = { + import cats.implicits.* + + (for { + decodedString <- scala.util + .Try { + val decodedBytes = java.util.Base64.getDecoder.decode(token) + new String(decodedBytes, StandardCharsets.UTF_8) + } + .toEither + .leftMap(_ => "Failed to decode base64 token") + decoded <- io.circe.parser.decode(decodedString)(codec).leftMap(_.getMessage) + } yield decoded).fold( + msg => + throw io.grpc.Status.INVALID_ARGUMENT + .withDescription(msg) + .asRuntimeException(), + identity, + ) + } + } + case class QueryAcsSnapshotResult( migrationId: Long, snapshotRecordTime: CantonTimestamp, createdEventsInPage: Vector[SpliceCreatedEvent], - afterToken: Option[Long], + afterToken: Option[QueryAcsSnapshotPaginationToken], ) private val amuletQualifiedName = diff --git a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/AppActivityStore.scala b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/AppActivityStore.scala index 2b72c75e71..cba770a875 100644 --- a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/AppActivityStore.scala +++ b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/AppActivityStore.scala @@ -12,21 +12,16 @@ import scala.concurrent.Future */ trait AppActivityStore { - /** Find the earliest round for which all app activity records have been ingested. + /** Ingestion status for a specific round, used by the Scan HTTP + * endpoints when no root hash or activity totals are yet stored. */ - def earliestRoundWithCompleteAppActivity()(implicit + def ingestionStatusForRound(roundNumber: Long)(implicit tc: TraceContext - ): Future[Option[Long]] + ): Future[AppActivityStore.RoundIngestionStatus] - /** The earliest round for which we have ingested app activity records. - * This round may not have all app activity records ingested. - * - * Returns None if no app activity records have been ingested. - * - * Return -1 for the first SV, if the ingestion started from beginning of round 0, - * indicating that this SV has complete data of round 0. + /** Find the earliest round for which all app activity records have been ingested. */ - def earliestIngestedRound()(implicit + def earliestRoundWithCompleteAppActivity()(implicit tc: TraceContext ): Future[Option[Long]] @@ -39,3 +34,24 @@ trait AppActivityStore { /** The record time of the first activity record in the store. */ def startedIngestingAt(implicit tc: TraceContext): Future[Option[Long]] } + +object AppActivityStore { + + /** Whether this Scan can ever be authoritative for a given round, + * or whether the answer will arrive as ingestion catches up. + */ + sealed trait RoundIngestionStatus + + object RoundIngestionStatus { + + /** Cannot compute an answer for this round from local state. + * Callers should delegate to BFT read. + */ + case object CannotProvide extends RoundIngestionStatus + + /** Do not yet have an answer but expect to have one after + * ingesting up to this round. Callers should retry. + */ + case object Undetermined extends RoundIngestionStatus + } +} diff --git a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/CachingScanRewardsReferenceStore.scala b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/CachingScanRewardsReferenceStore.scala index 244ae565e4..9dda395ffd 100644 --- a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/CachingScanRewardsReferenceStore.scala +++ b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/CachingScanRewardsReferenceStore.scala @@ -10,7 +10,12 @@ import com.digitalasset.canton.tracing.TraceContext import com.github.blemale.scaffeine.Scaffeine import org.lfdecentralizedtrust.splice.codegen.java.splice.amulet.rewardaccountingv2.CalculateRewardsV2 import org.lfdecentralizedtrust.splice.codegen.java.splice.round.OpenMiningRound -import org.lfdecentralizedtrust.splice.store.{Limit, MultiDomainAcsStore, SynchronizerStore} +import org.lfdecentralizedtrust.splice.store.{ + Limit, + MultiDomainAcsStore, + SynchronizerStore, + TimestampWithMigrationId, +} import org.lfdecentralizedtrust.splice.util.Contract import scala.concurrent.{ExecutionContext, Future} @@ -52,7 +57,7 @@ class CachingScanRewardsReferenceStore private[splice] ( override def lookupActiveOpenMiningRounds( recordTimes: Seq[CantonTimestamp] - )(implicit tc: TraceContext): Future[Map[CantonTimestamp, (Long, CantonTimestamp)]] = + )(implicit tc: TraceContext): Future[Map[CantonTimestamp, TimestampWithMigrationId]] = store.lookupActiveOpenMiningRounds(recordTimes) override def lookupFeaturedAppPartiesAsOf( diff --git a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/ScanEventStore.scala b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/ScanEventStore.scala index 4729fce4ef..d7d378c32b 100644 --- a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/ScanEventStore.scala +++ b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/ScanEventStore.scala @@ -10,7 +10,7 @@ import org.lfdecentralizedtrust.splice.scan.store.db.{DbAppActivityRecordStore, import org.lfdecentralizedtrust.splice.store.TreeUpdateWithMigrationId import org.lfdecentralizedtrust.splice.store.UpdateHistory import com.digitalasset.canton.data.CantonTimestamp -import org.lfdecentralizedtrust.splice.store.PageLimit +import org.lfdecentralizedtrust.splice.store.{PageLimit, TimestampWithMigrationId} import scala.collection.immutable.SortedMap import scala.concurrent.{ExecutionContext, Future} @@ -60,7 +60,7 @@ class ScanEventStore( } def getEvents( - afterO: Option[(Long, CantonTimestamp)], + afterO: Option[TimestampWithMigrationId], currentMigrationId: Long, limit: PageLimit, )(implicit tc: TraceContext): Future[Seq[Event]] = { @@ -88,9 +88,9 @@ class ScanEventStore( verdictStore.listTransactionViews(v.rowId).map(views => v -> views) ) } yield { - val verdictEntries: Iterator[((Long, CantonTimestamp), Verdict)] = + val verdictEntries: Iterator[(TimestampWithMigrationId, Verdict)] = verdictsWithViews.iterator.map { case (v, views) => - val k = (v.migrationId, v.recordTime) + val k = TimestampWithMigrationId(v.recordTime, v.migrationId) k -> (v -> views) } @@ -100,11 +100,11 @@ class ScanEventStore( val mergedSorted = { val fromUpdates = filteredUpdates.iterator.foldLeft( SortedMap.empty[ - (Long, CantonTimestamp), + TimestampWithMigrationId, (Option[Verdict], Option[TreeUpdateWithMigrationId]), ] ) { case (acc, u) => - val k = (u.migrationId, u.update.update.recordTime) + val k = TimestampWithMigrationId(u.update.update.recordTime, u.migrationId) acc.updated(k, (None, Some(u))) } verdictEntries.foldLeft(fromUpdates) { case (acc, (k, v)) => @@ -151,12 +151,12 @@ class ScanEventStore( // Filtering logic extracted out for unit testing object ScanEventStore { def allowF( - afterO: Option[(Long, CantonTimestamp)], + afterO: Option[TimestampWithMigrationId], currentMigrationId: Long, currentMigrationCap: CantonTimestamp, )(mig: Long, rt: CantonTimestamp): Boolean = { afterO match { - case Some((afterMig, afterRt)) if mig == afterMig => + case Some(TimestampWithMigrationId(afterRt, afterMig)) if mig == afterMig => if (mig < currentMigrationId) rt > afterRt else rt > afterRt && rt <= currentMigrationCap case _ if mig < currentMigrationId => diff --git a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/ScanRewardsReferenceStore.scala b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/ScanRewardsReferenceStore.scala index e98796e306..56e7e93f8b 100644 --- a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/ScanRewardsReferenceStore.scala +++ b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/ScanRewardsReferenceStore.scala @@ -16,7 +16,12 @@ import org.lfdecentralizedtrust.splice.codegen.java.splice.round.OpenMiningRound import org.lfdecentralizedtrust.splice.config.IngestionConfig import org.lfdecentralizedtrust.splice.environment.RetryProvider import org.lfdecentralizedtrust.splice.scan.store.db.ScanRewardsReferenceTables.ScanRewardsReferenceStoreRowData -import org.lfdecentralizedtrust.splice.store.{AppStore, Limit, MultiDomainAcsStore} +import org.lfdecentralizedtrust.splice.store.{ + AppStore, + Limit, + MultiDomainAcsStore, + TimestampWithMigrationId, +} import org.lfdecentralizedtrust.splice.store.db.AcsInterfaceViewRowData import org.lfdecentralizedtrust.splice.util.{Contract, TemplateJsonDecoder} @@ -57,7 +62,7 @@ trait ScanRewardsReferenceStore extends AppStore { */ def lookupActiveOpenMiningRounds( recordTimes: Seq[CantonTimestamp] - )(implicit tc: TraceContext): Future[Map[CantonTimestamp, (Long, CantonTimestamp)]] + )(implicit tc: TraceContext): Future[Map[CantonTimestamp, TimestampWithMigrationId]] def lookupFeaturedAppPartiesAsOf( asOf: CantonTimestamp diff --git a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/AcsSnapshotBulkStorage.scala b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/AcsSnapshotBulkStorage.scala index 3adb6e4652..57928ca3f5 100644 --- a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/AcsSnapshotBulkStorage.scala +++ b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/AcsSnapshotBulkStorage.scala @@ -79,6 +79,16 @@ class AcsSnapshotBulkStoragePersistentProgress( kvProvider.store.readValueAndLogOnDecodingFailure(firstSnapshotKvStoreKey).value } + def reset(implicit + tc: TraceContext, + ec: ExecutionContext, + ): Future[Unit] = { + for { + _ <- kvProvider.store.deleteKey(latestSnapshotKvStoreKey) + _ <- kvProvider.store.deleteKey(firstSnapshotKvStoreKey) + } yield {} + } + def persistLatestProcessedSnapshotTimestamp(ts: TimestampWithMigrationId)(implicit tc: TraceContext, ec: ExecutionContext, diff --git a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/AcsSnapshotBulkStorageCommitFromStaging.scala b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/AcsSnapshotBulkStorageCommitFromStaging.scala index 286f0a18a4..7f43f33eea 100644 --- a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/AcsSnapshotBulkStorageCommitFromStaging.scala +++ b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/AcsSnapshotBulkStorageCommitFromStaging.scala @@ -21,6 +21,7 @@ class AcsSnapshotBulkStorageCommitFromStaging( bulkStorageReader: BulkStorageReader, appConfig: BulkStorageConfig, scanConnection: PeerBftScanConnection, + onObjectCommitted: Seq[S3BucketConnection.ObjectKeyAndChecksum] => Unit, val loggerFactory: NamedLoggerFactory, )(implicit ec: ExecutionContextExecutor @@ -60,6 +61,7 @@ class AcsSnapshotBulkStorageCommitFromStaging( appConfig, scanConnection, loggerFactory, + onObjectCommitted, ) } } diff --git a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/BulkStorage.scala b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/BulkStorage.scala index 169bc8e99d..5f7586df12 100644 --- a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/BulkStorage.scala +++ b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/BulkStorage.scala @@ -22,6 +22,7 @@ import org.lfdecentralizedtrust.splice.scan.store.{ import org.lfdecentralizedtrust.splice.store.{HistoryMetrics, S3BucketConnection, UpdateHistory} import scala.concurrent.{ExecutionContextExecutor, Future} +import com.digitalasset.canton.discard.Implicits.DiscardOps import cats.implicits.* import org.apache.pekko.stream.scaladsl.Source import org.lfdecentralizedtrust.splice.PekkoRetryableService @@ -152,6 +153,16 @@ class BulkStorage( reader, appConfig, scanConnection, + objs => + objs.foreach { obj => + val encoding = ScanStorageConfig.Encoding.all.toList + .collectFirst { + case enc if enc.storageKeyRegex("ACS").matches(obj.key) => + enc.key + } + .getOrElse("unknown") + historyMetrics.BulkStorage.incAcsSnapshotObjects(encoding, "committed") + }, loggerFactory, ) val acsCommitted = new AcsSnapshotBulkStorage( @@ -185,6 +196,16 @@ class BulkStorage( reader, appConfig, scanConnection, + objs => + objs.foreach { obj => + val encoding = ScanStorageConfig.Encoding.all.toList + .collectFirst { + case enc if enc.storageKeyRegex("updates").matches(obj.key) => + enc.key + } + .getOrElse("unknown") + historyMetrics.BulkStorage.incUpdateObjects(encoding, "committed") + }, loggerFactory, ) val updatesCommitted = new UpdateHistoryBulkStorage( @@ -196,10 +217,30 @@ class BulkStorage( loggerFactory, ) - private val services = + // Services are only started once initialization has completed. + private lazy val services = Seq[PekkoRetryableService[?]](acsStaging, acsCommitted, updatesStaging, updatesCommitted) .map(_.asPekkoRetryingService(automationConfig, backoffClock, retryProvider)) + private def initialize(): Future[BulkStorage] = { + val resetAll = + if (appConfig.debugForceStartFromGenesis) { + logger.warn( + "debugForceStartFromGenesis is set to true, resetting all bulk storage progress and starting from genesis" + ) + for { + _ <- acsStagingProgress.reset + _ <- acsCommittedProgress.reset + _ <- updatesStagingProgress.reset + _ <- updatesCommittedProgress.reset + } yield () + } else Future.unit + resetAll.map { _ => + services.discard + this + } + } + final override def closeAsync(): Seq[AsyncOrSyncCloseable] = { LifeCycle.close(scanConnection)(logger) services.flatMap(_.closeAsync()) @@ -237,7 +278,7 @@ object BulkStorage { tracer: Tracer, httpClient: HttpClient, templateJsonDecoder: TemplateJsonDecoder, - ): BulkStorage = { + ): Future[BulkStorage] = { val logger = loggerFactory.getTracedLogger(classOf[BulkStorage]) (appConfig.staging, appConfig.committed).tupled.fold { @@ -264,7 +305,7 @@ object BulkStorage { upgradesConfig, retryProvider, loggerFactory, - ) + ).initialize() } } } diff --git a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/BulkStorageCommitFromStaging.scala b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/BulkStorageCommitFromStaging.scala index 0faaff5139..5d1671aeca 100644 --- a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/BulkStorageCommitFromStaging.scala +++ b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/BulkStorageCommitFromStaging.scala @@ -25,6 +25,7 @@ class BulkStorageCommitFromStaging[T]( appConfig: BulkStorageConfig, scanConnection: PeerBftScanConnection, override val loggerFactory: NamedLoggerFactory, + onObjectCommitted: Seq[ObjectKeyAndChecksum] => Unit = _ => (), )(implicit tc: TraceContext, ec: ExecutionContextExecutor, @@ -57,18 +58,76 @@ class BulkStorageCommitFromStaging[T]( logger.debug( s"Consensus achieved on ${consensusChecksums.length} out of ${objects.length} objects" ) - val consensus = - bftChecksums.checksums.filter(_.value.isDefined).map(_.value) == objects.map(oc => - Some(oc.checksum) + + if (consensusChecksums.length < objects.length) { + logger.debug( + s"Not all objects are known to the BFT peers yet. Will retry after delay." ) - if (consensusChecksums.length == objects.length && !consensus) { - logger.error( - s"All objects are known to the BFT peers, but the checksums do not match. This indicates an error in the actual data generated for bulk storage. Expected: ${objects - .map(_.checksum) - .mkString(", ")}, got: ${consensusChecksums.mkString(", ")}" + false + } else { + logger.debug( + s"All objects are known to the BFT peers. Checking if checksums match." ) + val consensus = + bftChecksums.checksums.filter(_.value.isDefined).map(_.value) == objects.map(oc => + Some(oc.checksum) + ) + if (!consensus) { + logger.error( + s"Checksums do not match for objects ${objects.map(_.key).mkString(", ")}. My checksums are: ${objects + .map(_.checksum) + .mkString(", ")}, consensus checksums are: ${consensusChecksums.mkString(", ")}" + ) + + if (appConfig.debugObjectsToNotCommit.intersect(objects.map(_.key)).nonEmpty) { + logger.debug( + s"Some relevant objects are listed in debugObjectsToNotCommit, will ignore them for the consensus check. Ignored objects: ${appConfig.debugObjectsToNotCommit + .intersect(objects.map(_.key)) + .mkString(", ")}" + ) + val objectsWithConsensusChecksums = objects.zip(consensusChecksums) + // Filter out objects for which the key is listed in appConfig.debugObjectsToNotCommit + val unignoredObjectsWithTheirConsensusChecksums = + objectsWithConsensusChecksums.filter { case (obj, _) => + !appConfig.debugObjectsToNotCommit.contains(obj.key) + } + val unignoredObjectsWithMyChecksums = + objects.filter(obj => !appConfig.debugObjectsToNotCommit.contains(obj.key)) + // recheck consensus, but now only on the unignored objects. The comparison should be similar to val consensus above + val unignoredConsensus = + unignoredObjectsWithTheirConsensusChecksums + .filter(_._2.value.isDefined) + .map(_._2.value) == unignoredObjectsWithMyChecksums.map(oc => + Some(oc.checksum) + ) + + if (!unignoredConsensus) { + logger.error( + s"Checksums still do not match for unignored objects ${unignoredObjectsWithMyChecksums + .map(_.key) + .mkString(", ")}. Expected: ${unignoredObjectsWithMyChecksums + .map(_.checksum) + .mkString(", ")}, got: ${unignoredObjectsWithTheirConsensusChecksums.map(_._2.value).mkString(", ")}" + ) + } else { + logger.debug( + s"After ignoring objects from the config, Checksums match ${unignoredObjectsWithMyChecksums.map(_.key).mkString(", ")}. Proceeding with commit." + ) + } + unignoredConsensus + } else { + logger.trace( + s"No relevant objects are listed in debugObjectsToNotCommit, will not ignore any objects for the consensus check." + ) + consensus + } + } else { + logger.trace( + s"Checksums match for all objects ${objects.map(_.key).mkString(", ")}. Proceeding with commit." + ) + true + } } - consensus case None => false } @@ -78,7 +137,6 @@ class BulkStorageCommitFromStaging[T]( Future.successful(true) } } - // TODO(#5884): implement the BFT check private def waitForBftAgreement: Flow[ (T, Seq[ObjectKeyAndChecksum]), @@ -120,8 +178,15 @@ class BulkStorageCommitFromStaging[T]( ) Future.unit case false => - logger.debug(s"Copying object ${obj.key} from staging to committed storage") - committedS3Connection.copyObject(stagingS3Connection.bucketName, obj.key) + if (appConfig.debugObjectsToNotCommit.contains(obj.key)) { + logger.debug( + s"Object ${obj.key} is listed in debugObjectsToNotCommit, skipping copy to committed storage" + ) + Future.unit + } else { + logger.debug(s"Copying object ${obj.key} from staging to committed storage") + committedS3Connection.copyObject(stagingS3Connection.bucketName, obj.key) + } } } @@ -137,7 +202,10 @@ class BulkStorageCommitFromStaging[T]( ) Future .sequence(objs.map(copyObjectToCommitted(stagingS3Connection, committedS3Connection))) - .map(_ => (ts, objs)) + .map { _ => + onObjectCommitted(objs) + (ts, objs) + } } private def deleteFromStaging: Flow[ @@ -180,6 +248,7 @@ object BulkStorageCommitFromStaging { appConfig: BulkStorageConfig, scanConnection: PeerBftScanConnection, loggerFactory: NamedLoggerFactory, + onObjectCommitted: Seq[ObjectKeyAndChecksum] => Unit = _ => (), )(implicit tc: TraceContext, ec: ExecutionContextExecutor, @@ -191,6 +260,7 @@ object BulkStorageCommitFromStaging { appConfig, scanConnection, loggerFactory, + onObjectCommitted, ).getFlow } } diff --git a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/GroupedWeightS3ObjectFlow.scala b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/GroupedWeightS3ObjectFlow.scala index 60df59359d..9825e0e6b4 100644 --- a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/GroupedWeightS3ObjectFlow.scala +++ b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/GroupedWeightS3ObjectFlow.scala @@ -83,6 +83,14 @@ case class GroupedWeightS3ObjectFlow( @SuppressWarnings(Array("org.wartremover.warts.Var")) private var state = State.initial() + // Guards against finishing the same object twice, which can otherwise happen because finish() + // is triggered both from uploadCallback and from onUpstreamFinish. + // At the time of writing, a duplicate finish is actually harmless, as it calls + // AppendWriteObject.finish() which is idempotent. We guard against it anyway, to protect against + // future changes that would make finishing an object non-idempotent. + @SuppressWarnings(Array("org.wartremover.warts.Var")) + private var finishingObject = false + private def objectDone = state.currentObjectSize >= maxObjectSize || isClosed(in) private val uploadCallback = getAsyncCallback[Unit] { _ => @@ -111,6 +119,7 @@ case class GroupedWeightS3ObjectFlow( private val finishCallback = getAsyncCallback[Unit] { _ => logger.debug(s"Finished uploading and finalizing object ${state.currentObject.key}") + finishingObject = false push(out, state.currentObject.key) if (isClosed(in)) { logger.trace("Upstream completed, completing too.") @@ -156,13 +165,24 @@ case class GroupedWeightS3ObjectFlow( } private def finishCurrentObject(): Unit = - state.currentObject.finish().onComplete { - case Success(_) => finishCallback.invoke(()) - case Failure(ex) => failCallback.invoke(ex) + if (finishingObject) { + logger.debug( + s"Object ${state.currentObject.key} is already being finished, not finishing it again" + ) + } else { + finishingObject = true + state.currentObject.finish().onComplete { + case Success(_) => finishCallback.invoke(()) + case Failure(ex) => failCallback.invoke(ex) + } } override def onUpstreamFinish(): Unit = { - if (state.numPendingPartUploads == 0) { + if (finishingObject) { + logger.debug( + s"Upstream finished while object ${state.currentObject.key} is being finished, waiting for it to complete" + ) + } else if (state.numPendingPartUploads == 0) { if (state.currentObjectSize > 0) { logger.debug( s"Upstream finished, finishing current object ${state.currentObject.key} with size ${state.currentObjectSize}" diff --git a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/SingleAcsSnapshotBulkStorage.scala b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/SingleAcsSnapshotBulkStorage.scala index bddfca8da7..5c54c8ee1a 100644 --- a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/SingleAcsSnapshotBulkStorage.scala +++ b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/SingleAcsSnapshotBulkStorage.scala @@ -33,7 +33,7 @@ object Position { case object End extends Position - final case class Index(value: Long) extends Position + final case class Index(value: AcsSnapshotStore.QueryAcsSnapshotPaginationToken) extends Position } class SingleAcsSnapshotBulkStorage( @@ -49,7 +49,7 @@ class SingleAcsSnapshotBulkStorage( private def getAcsSnapshotChunk( timestamp: TimestampWithMigrationId, - after: Option[Long], + after: Option[AcsSnapshotStore.QueryAcsSnapshotPaginationToken], ): Future[(Position, Vector[SpliceCreatedEvent])] = { for { snapshot <- acsSnapshotStore.queryAcsSnapshot( @@ -107,7 +107,7 @@ class SingleAcsSnapshotBulkStorage( .storageKey("ACS", objIdx)}", loggerFactory, ), - encoding => historyMetrics.BulkStorage.incAcsSnapshotObjects(encoding.key), + encoding => historyMetrics.BulkStorage.incAcsSnapshotObjects(encoding.key, "staging"), ) ) .fold(Seq.empty[String])(_ :+ _) diff --git a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/UpdateHistoryBulkStorage.scala b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/UpdateHistoryBulkStorage.scala index a27d418a20..d0e858829e 100644 --- a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/UpdateHistoryBulkStorage.scala +++ b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/UpdateHistoryBulkStorage.scala @@ -67,6 +67,10 @@ class UpdateHistoryBulkStoragePersistentProgress( ) }) } + + def reset(implicit tc: TraceContext): Future[Unit] = { + kvProvider.store.deleteKey(kvStoreKey) + } } /** An abstract class for pipelines that process update history for bulk storage. diff --git a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/UpdateHistoryBulkStorageCommitFromStaging.scala b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/UpdateHistoryBulkStorageCommitFromStaging.scala index 5965a09c85..7a07c2f326 100644 --- a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/UpdateHistoryBulkStorageCommitFromStaging.scala +++ b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/UpdateHistoryBulkStorageCommitFromStaging.scala @@ -20,6 +20,7 @@ class UpdateHistoryBulkStorageCommitFromStaging( bulkStorageReader: BulkStorageReader, appConfig: BulkStorageConfig, scanConnection: PeerBftScanConnection, + onObjectCommitted: Seq[S3BucketConnection.ObjectKeyAndChecksum] => Unit, val loggerFactory: NamedLoggerFactory, )(implicit ec: ExecutionContextExecutor @@ -43,6 +44,7 @@ class UpdateHistoryBulkStorageCommitFromStaging( appConfig, scanConnection, loggerFactory, + onObjectCommitted, ) override def getNextSegmentAfter( diff --git a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/UpdateHistorySegmentBulkStorage.scala b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/UpdateHistorySegmentBulkStorage.scala index ea2561d8d4..fb8b915b96 100644 --- a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/UpdateHistorySegmentBulkStorage.scala +++ b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/UpdateHistorySegmentBulkStorage.scala @@ -63,7 +63,7 @@ class UpdateHistorySegmentBulkStorage( ): Future[Option[(TimestampWithMigrationId, Seq[TreeUpdateWithMigrationId])]] = { for { updates <- updateHistory.getUpdatesWithoutImportUpdates( - Some((afterTs.migrationId, afterTs.timestamp)), + Some(TimestampWithMigrationId(afterTs.timestamp, afterTs.migrationId)), PageLimit.tryCreate(storageConfig.bulkDbReadChunkSize), ) updatesInSegment = updates.filter(update => @@ -170,7 +170,7 @@ class UpdateHistorySegmentBulkStorage( loggerFactory, ) ), - encoding => historyMetrics.BulkStorage.incUpdateObjects(encoding.key), + encoding => historyMetrics.BulkStorage.incUpdateObjects(encoding.key, "staging"), ) ) .orElse(Source.lazySource { () => diff --git a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/db/DbAppActivityRecordStore.scala b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/db/DbAppActivityRecordStore.scala index 8f6595b203..60a3664e00 100644 --- a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/db/DbAppActivityRecordStore.scala +++ b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/db/DbAppActivityRecordStore.scala @@ -4,6 +4,7 @@ package org.lfdecentralizedtrust.splice.scan.store.db import org.lfdecentralizedtrust.splice.scan.store.AppActivityStore +import org.lfdecentralizedtrust.splice.scan.store.AppActivityStore.RoundIngestionStatus import org.lfdecentralizedtrust.splice.store.UpdateHistory import org.lfdecentralizedtrust.splice.util.FutureUnlessShutdownUtil.futureUnlessShutdownToFuture import com.digitalasset.canton.logging.{NamedLoggerFactory, NamedLogging} @@ -171,7 +172,7 @@ class DbAppActivityRecordStore( * This round may not have all app activity records ingested. * Returns None if no app activity records have been ingested, ie meta row does not exist. */ - def earliestIngestedRound()(implicit + private[store] def earliestIngestedRound()(implicit tc: TraceContext ): Future[Option[Long]] = { val codeVersion = ingestionVersions.code @@ -187,6 +188,32 @@ class DbAppActivityRecordStore( ) } + override def ingestionStatusForRound(roundNumber: Long)(implicit + tc: TraceContext + ): Future[RoundIngestionStatus] = + earliestIngestedRound().map { + case Some(earliestIngested) if roundNumber <= earliestIngested => + // We should have data for this round but no root hash exists: + // a peer likely does, so delegate. + RoundIngestionStatus.CannotProvide + + case Some(_) => + // Meta row present but round is beyond our ingested boundary — + // ingestion is still catching up; retry. + RoundIngestionStatus.Undetermined + + case None if !isFirstSv => + // Late-joining Scan with no ingestion boundary of its own — + // it might seem Undetermined is right, but peers do have one, + // so we delegate. + RoundIngestionStatus.CannotProvide + + case None => + // firstSV during initial ingestion (brief startup window before + // the meta row is inserted) — retry. + RoundIngestionStatus.Undetermined + } + /** Find the latest round with complete app activity. * A round is complete once the verdict ingestion has moved passed its archival. * Returns None if no meta row exists or archival of a round has not happened yet. diff --git a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/db/DbScanRewardsReferenceStore.scala b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/db/DbScanRewardsReferenceStore.scala index 09313f5e42..1dd5c85fdd 100644 --- a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/db/DbScanRewardsReferenceStore.scala +++ b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/db/DbScanRewardsReferenceStore.scala @@ -17,7 +17,12 @@ import org.lfdecentralizedtrust.splice.codegen.java.splice.round.OpenMiningRound import org.lfdecentralizedtrust.splice.config.IngestionConfig import org.lfdecentralizedtrust.splice.environment.RetryProvider import org.lfdecentralizedtrust.splice.scan.store.ScanRewardsReferenceStore -import org.lfdecentralizedtrust.splice.store.{Limit, LimitHelpers, TcsStore} +import org.lfdecentralizedtrust.splice.store.{ + Limit, + LimitHelpers, + TcsStore, + TimestampWithMigrationId, +} import org.lfdecentralizedtrust.splice.store.db.{ AcsArchiveConfig, AcsQueries, @@ -103,7 +108,7 @@ class DbScanRewardsReferenceStore( override def lookupActiveOpenMiningRounds( recordTimes: Seq[CantonTimestamp] - )(implicit tc: TraceContext): Future[Map[CantonTimestamp, (Long, CantonTimestamp)]] = { + )(implicit tc: TraceContext): Future[Map[CantonTimestamp, TimestampWithMigrationId]] = { tcsStore.getEarliestArchivedAt().flatMap { case None => Future.successful(Map.empty) @@ -125,7 +130,10 @@ class DbScanRewardsReferenceStore( .flatMap { r => val opensAt = CantonTimestamp.assertFromInstant(r.contract.payload.opensAt) Option.when(opensAt >= ingestionStart) { - recordTime -> (r.contract.payload.round.number.toLong, opensAt) + recordTime -> TimestampWithMigrationId( + opensAt, + r.contract.payload.round.number.toLong, + ) } } }.toMap diff --git a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/db/DbScanVerdictStore.scala b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/db/DbScanVerdictStore.scala index f14fca20d1..bd565c986e 100644 --- a/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/db/DbScanVerdictStore.scala +++ b/apps/scan/src/main/scala/org/lfdecentralizedtrust/splice/scan/store/db/DbScanVerdictStore.scala @@ -25,7 +25,7 @@ import slick.dbio.DBIO import java.util.concurrent.atomic.AtomicReference import scala.concurrent.{ExecutionContext, Future} import cats.data.NonEmptyList -import org.lfdecentralizedtrust.splice.store.UpdateHistory +import org.lfdecentralizedtrust.splice.store.{TimestampWithMigrationId, UpdateHistory} import org.lfdecentralizedtrust.splice.scan.store.db.DbAppActivityRecordStore.AppActivityRecordT object DbScanVerdictStore { @@ -565,14 +565,14 @@ class DbScanVerdictStore( ) private def afterFilters( - afterO: Option[(Long, CantonTimestamp)], + afterO: Option[TimestampWithMigrationId], includeImportUpdates: Boolean, ): NonEmptyList[SQLActionBuilder] = { val gt = if (includeImportUpdates) ">=" else ">" afterO match { case None => NonEmptyList.of(sql"migration_id >= 0 and record_time #$gt ${CantonTimestamp.MinValue}") - case Some((afterMigrationId, afterRecordTime)) => + case Some(TimestampWithMigrationId(afterRecordTime, afterMigrationId)) => NonEmptyList.of( sql"migration_id = ${afterMigrationId} and record_time > ${afterRecordTime} ", sql"migration_id > ${afterMigrationId} and record_time #$gt ${CantonTimestamp.MinValue}", @@ -615,7 +615,7 @@ class DbScanVerdictStore( } def listVerdicts( - afterO: Option[(Long, CantonTimestamp)], + afterO: Option[TimestampWithMigrationId], includeImportUpdates: Boolean, limit: Int, )(implicit tc: TraceContext): Future[Seq[VerdictT]] = { diff --git a/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/BftScanConnectionTest.scala b/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/BftScanConnectionTest.scala index 889fee4c45..1879ed49bb 100644 --- a/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/BftScanConnectionTest.scala +++ b/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/BftScanConnectionTest.scala @@ -97,6 +97,7 @@ class BftScanConnectionTest when(m.config).thenReturn( ScanAppClientConfig(NetworkAppClientConfig(scanUrl(n))) ) + when(m.url).thenReturn(Uri(scanUrl(n))) m } connections.foreach { connection => @@ -1043,7 +1044,7 @@ class BftScanConnectionTest connections.tail.foreach(c => when(c.getDsoPartyId()).thenReturn(delayedSuccess)) for { - result <- BftScanConnection.executeCall(call, connections, nTargetSuccess = 1, logger) + (result, _) <- BftScanConnection.executeCall(call, connections, nTargetSuccess = 1, logger) } yield result should be(partyIdA) } @@ -1111,7 +1112,7 @@ class BftScanConnectionTest } for { - result <- BftScanConnection.executeCall( + (result, _) <- BftScanConnection.executeCall( call, connections, nTargetSuccess = 2, @@ -1158,7 +1159,7 @@ class BftScanConnectionTest makeMockFail(connections(2), notFoundFailure) for { - result <- BftScanConnection.executeCall( + (result, _) <- BftScanConnection.executeCall( call, connections, nTargetSuccess = 2, @@ -1194,7 +1195,7 @@ class BftScanConnectionTest } for { - result <- BftScanConnection.executeCall( + (result, _) <- BftScanConnection.executeCall( call, connections, nTargetSuccess = 2, @@ -1245,26 +1246,36 @@ class BftScanConnectionTest // With n=4, we query only two connections randomly, and even with // retries a single call can fail to reach consensus. - def attempt(remaining: Int): Future[GetRewardAccountingRootHashResponse] = - bft.getRewardAccountingRootHash(round).flatMap { - case ok: GetRewardAccountingRootHashResponse.members.RewardAccountingRootHashOk => - Future.successful(ok) + def attempt(remaining: Int): Future[(GetRewardAccountingRootHashResponse, List[Uri])] = + bft.getRewardAccountingRootHashWithScanUris(round).flatMap { + case (ok: GetRewardAccountingRootHashResponse.members.RewardAccountingRootHashOk, uris) => + Future.successful((ok, uris)) case _ if remaining > 1 => attempt(remaining - 1) case other => Future.successful(other) } + // A call that reaches consensus here always queries a third scan that + // disagrees (returns IgnoreResponse or fails), which BftScanConnection + // logs at WARN for the reward-read paths. Assert that WARN is produced + // and suppress it so it doesn't fail the `sbt checkErrors` log-scan gate. loggerFactory - .assertEventuallyLogsSeq(SuppressionRule.LevelAndAbove(Level.INFO))( + .assertEventuallyLogsSeq(SuppressionRule.Level(Level.WARN))( attempt(100).map { resp => inside(resp) { - case GetRewardAccountingRootHashResponse.members.RewardAccountingRootHashOk(ok) => + case ( + GetRewardAccountingRootHashResponse.members.RewardAccountingRootHashOk(ok), + uris, + ) => ok.rootHash should be("aabb") ok.roundNumber should be(round) + uris.size should be(2) } }, logs => - logs.exists(l => - l.level == Level.INFO && l.message.contains("Reached consensus from") + logs.exists(log => + log.level == Level.WARN && log.message.contains( + "disagreed with consensus" + ) ) should be(true), ) .map(_ => succeed) @@ -1366,29 +1377,42 @@ class BftScanConnectionTest // With n=4, we query only two connections randomly, and even with // retries a single call can fail to reach consensus. - def attempt(remaining: Int): Future[GetRewardAccountingActivityTotalsResponse] = - bft.getRewardAccountingActivityTotals(round).flatMap { - case ok: GetRewardAccountingActivityTotalsResponse.members.RewardAccountingActivityTotalsOk => - Future.successful(ok) + def attempt(remaining: Int): Future[(GetRewardAccountingActivityTotalsResponse, List[Uri])] = + bft.getRewardAccountingActivityTotalsWithScanUris(round).flatMap { + case ( + ok: GetRewardAccountingActivityTotalsResponse.members.RewardAccountingActivityTotalsOk, + uris, + ) => + Future.successful((ok, uris)) case _ if remaining > 1 => attempt(remaining - 1) case other => Future.successful(other) } + // A call that reaches consensus here always queries a third scan that + // disagrees (returns IgnoreResponse or fails), which BftScanConnection + // logs at WARN for the reward-read paths. Assert that WARN is produced + // and suppress it so it doesn't fail the `sbt checkErrors` log-scan gate. loggerFactory - .assertEventuallyLogsSeq(SuppressionRule.LevelAndAbove(Level.INFO))( + .assertEventuallyLogsSeq(SuppressionRule.Level(Level.WARN))( attempt(100).map { resp => inside(resp) { - case GetRewardAccountingActivityTotalsResponse.members - .RewardAccountingActivityTotalsOk(ok) => + case ( + GetRewardAccountingActivityTotalsResponse.members + .RewardAccountingActivityTotalsOk(ok), + uris, + ) => ok.roundNumber should be(round) ok.totalAppActivityWeight should be(100L) ok.activePartiesCount should be(10L) ok.activityRecordsCount should be(5L) + uris.size should be(2) } }, logs => - logs.exists(l => - l.level == Level.INFO && l.message.contains("Reached consensus from") + logs.exists(log => + log.level == Level.WARN && log.message.contains( + "disagreed with consensus" + ) ) should be(true), ) .map(_ => succeed) diff --git a/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/SingleScanConnectionTest.scala b/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/SingleScanConnectionTest.scala new file mode 100644 index 0000000000..9852669a1c --- /dev/null +++ b/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/admin/api/client/SingleScanConnectionTest.scala @@ -0,0 +1,62 @@ +// Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package org.lfdecentralizedtrust.splice.scan.admin.api.client + +import com.digitalasset.canton.BaseTest +import io.circe.Json +import org.apache.pekko.http.scaladsl.model.{HttpRequest, StatusCodes} +import org.apache.pekko.stream.StreamTcpException +import org.lfdecentralizedtrust.splice.admin.api.client.commands.HttpCommandException +import org.lfdecentralizedtrust.splice.environment.BaseAppConnection +import org.scalatest.wordspec.AnyWordSpec + +class SingleScanConnectionTest extends AnyWordSpec with BaseTest { + + "SingleScanConnection.httpStatusLabel" should { + + "extract the status code of an unexpected JSON response" in { + SingleScanConnection.httpStatusLabel( + new BaseAppConnection.UnexpectedHttpJsonResponse(StatusCodes.NotFound, Json.obj()) + ) should be("404") + } + + "extract the status code of an unexpected malformed JSON response" in { + SingleScanConnection.httpStatusLabel( + new BaseAppConnection.UnexpectedHttpMalformedJsonResponse( + StatusCodes.BadGateway, + "not json", + ) + ) should be("502") + } + + "extract the status code of an unexpected text response" in { + SingleScanConnection.httpStatusLabel( + new BaseAppConnection.UnexpectedHttpTextResponse(StatusCodes.ServiceUnavailable, "nope") + ) should be("503") + } + + "extract the status code of an unexpected non-JSON response" in { + SingleScanConnection.httpStatusLabel( + new BaseAppConnection.UnexpectedHttpNonJsonResponse(StatusCodes.InternalServerError) + ) should be("500") + } + + "extract the status code of an HttpCommandException" in { + SingleScanConnection.httpStatusLabel( + HttpCommandException( + HttpRequest(), + StatusCodes.TooManyRequests, + HttpCommandException.RawResponse("slow down"), + ) + ) should be("429") + } + + "report 'none' for failures without an HTTP status code" in { + SingleScanConnection.httpStatusLabel( + new StreamTcpException("connection refused") + ) should be("none") + SingleScanConnection.httpStatusLabel(new RuntimeException("boom")) should be("none") + } + } +} diff --git a/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/rewards/AppActivityComputationTest.scala b/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/rewards/AppActivityComputationTest.scala index ef1bba310c..9f8c350492 100644 --- a/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/rewards/AppActivityComputationTest.scala +++ b/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/rewards/AppActivityComputationTest.scala @@ -11,6 +11,7 @@ import com.google.protobuf.timestamp.Timestamp as ProtoTimestamp import com.google.protobuf.ByteString import org.lfdecentralizedtrust.splice.scan.store.ScanRewardsReferenceStore import org.lfdecentralizedtrust.splice.scan.store.db.{DbAppActivityRecordStore, DbScanVerdictStore} +import org.lfdecentralizedtrust.splice.store.TimestampWithMigrationId import org.scalatest.wordspec.AnyWordSpec import scala.concurrent.Future @@ -219,7 +220,7 @@ class AppActivityComputationTest extends AnyWordSpec with BaseTest { val store = mock[ScanRewardsReferenceStore] when(store.lookupActiveOpenMiningRounds(any[Seq[CantonTimestamp]])(any[TraceContext])) .thenAnswer { (times: Seq[CantonTimestamp]) => - Future.successful(times.map(_ -> (0L, roundOpensAt)).toMap) + Future.successful(times.map(_ -> TimestampWithMigrationId(roundOpensAt, 0L)).toMap) } when(store.lookupFeaturedAppPartiesAsOf(any[CantonTimestamp])(any[TraceContext])) .thenReturn(Future.successful(featuredWeights)) diff --git a/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/DbAppActivityRecordStoreTest.scala b/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/DbAppActivityRecordStoreTest.scala index 7d14f778c4..30a9867878 100644 --- a/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/DbAppActivityRecordStoreTest.scala +++ b/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/DbAppActivityRecordStoreTest.scala @@ -7,6 +7,7 @@ import com.digitalasset.canton.topology.SynchronizerId import com.digitalasset.canton.tracing.TraceContext import com.digitalasset.canton.resource.DbStorage import com.digitalasset.canton.lifecycle.FutureUnlessShutdown +import org.lfdecentralizedtrust.splice.scan.store.AppActivityStore.RoundIngestionStatus import org.lfdecentralizedtrust.splice.scan.store.db.DbAppActivityRecordStore import org.lfdecentralizedtrust.splice.scan.store.db.DbAppActivityRecordStore.* import org.lfdecentralizedtrust.splice.scan.store.db.DbScanVerdictStore @@ -657,6 +658,51 @@ class DbAppActivityRecordStoreTest } } + "ingestionStatusForRound" should { + + "return CannotProvide when meta row absent and isFirstSv=false" in { + for { + (store, _) <- newStore(isFirstSv = false) + result <- store.ingestionStatusForRound(5L) + } yield { + result shouldBe RoundIngestionStatus.CannotProvide + } + } + + "return Undetermined when meta row absent and isFirstSv=true" in { + for { + (store, _) <- newStore(isFirstSv = true) + result <- store.ingestionStatusForRound(5L) + } yield { + result shouldBe RoundIngestionStatus.Undetermined + } + } + + "return CannotProvide when meta row present and roundNumber <= earliestIngested" in { + for { + (store, _) <- newStore() + baseTs = CantonTimestamp.now() + _ <- store.insertActivityRecordMetaForTesting(1, 0, baseTs.toMicros, 10L, Some(11L)) + atBoundary <- store.ingestionStatusForRound(10L) + below <- store.ingestionStatusForRound(5L) + } yield { + atBoundary shouldBe RoundIngestionStatus.CannotProvide + below shouldBe RoundIngestionStatus.CannotProvide + } + } + + "return Undetermined when meta row present and roundNumber > earliestIngested" in { + for { + (store, _) <- newStore() + baseTs = CantonTimestamp.now() + _ <- store.insertActivityRecordMetaForTesting(1, 0, baseTs.toMicros, 10L, Some(11L)) + result <- store.ingestionStatusForRound(15L) + } yield { + result shouldBe RoundIngestionStatus.Undetermined + } + } + } + "lookupActivityRecordMeta" should { "return None when no meta row exists" in { diff --git a/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/QueryAcsSnapshotPaginationTokenTest.scala b/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/QueryAcsSnapshotPaginationTokenTest.scala new file mode 100644 index 0000000000..0b5c5da666 --- /dev/null +++ b/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/QueryAcsSnapshotPaginationTokenTest.scala @@ -0,0 +1,51 @@ +// Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package org.lfdecentralizedtrust.splice.scan.store + +import com.digitalasset.canton.BaseTest +import org.lfdecentralizedtrust.splice.scan.store.AcsSnapshotStore.QueryAcsSnapshotPaginationToken +import org.lfdecentralizedtrust.splice.scan.store.AcsSnapshotStore.QueryAcsSnapshotPaginationToken.RowIdQueryAcsSnapshotPaginationToken +import org.scalatest.wordspec.AnyWordSpec +import scala.util.Try + +class QueryAcsSnapshotPaginationTokenTest extends AnyWordSpec with BaseTest { + + "RowIdQueryAcsSnapshotPaginationToken" should { + + "encode to base64 and decode back" in { + val token = RowIdQueryAcsSnapshotPaginationToken(42L) + val encoded = token.encodeToBase64 + val decoded = QueryAcsSnapshotPaginationToken.tryDecodeFromBase64(encoded) + decoded shouldBe token + } + + "produce different encoded values for different row ids" in { + val token1 = RowIdQueryAcsSnapshotPaginationToken(1L) + val token2 = RowIdQueryAcsSnapshotPaginationToken(2L) + token1.encodeToBase64 should not equal token2.encodeToBase64 + } + } + + "QueryAcsSnapshotPaginationToken.decodeFromBase64" should { + + "return Left for an invalid base64 string" in { + val result = Try(QueryAcsSnapshotPaginationToken.tryDecodeFromBase64("not-valid-base64!!!")) + result.isFailure should be(true) + } + + "return Left for valid base64 but invalid JSON content" in { + val encoded = java.util.Base64.getEncoder.encodeToString("not-a-long".getBytes("UTF-8")) + val result = Try(QueryAcsSnapshotPaginationToken.tryDecodeFromBase64(encoded)) + result.isFailure should be(true) + } + + "return Left for valid base64 with JSON object instead of long" in { + val encoded = + java.util.Base64.getEncoder.encodeToString("""{"after": 42}""".getBytes("UTF-8")) + val result = Try(QueryAcsSnapshotPaginationToken.tryDecodeFromBase64(encoded)) + result.isFailure should be(true) + } + } + +} diff --git a/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/ScanEventStoreTest.scala b/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/ScanEventStoreTest.scala index de8dc884af..f066410b23 100644 --- a/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/ScanEventStoreTest.scala +++ b/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/ScanEventStoreTest.scala @@ -9,6 +9,7 @@ import org.lfdecentralizedtrust.splice.store.{ HistoryMetrics, PageLimit, StoreTestBase, + TimestampWithMigrationId, UpdateHistory, } import org.lfdecentralizedtrust.splice.scan.store.db.{DbAppActivityRecordStore, DbScanVerdictStore} @@ -122,14 +123,14 @@ class ScanEventStoreTest extends StoreTestBase with HasExecutionContext with Spl // Fetch with cursor events2 <- fetchEvents( ctx.eventStore, - Some((domainMigrationId, recordTs1.minusSeconds(1))), + Some(TimestampWithMigrationId(recordTs1.minusSeconds(1), domainMigrationId)), domainMigrationId, pageLimit, ) // Fetch after recordTs1 events3 <- fetchEvents( ctx.eventStore, - Some((domainMigrationId, recordTs1)), + Some(TimestampWithMigrationId(recordTs1, domainMigrationId)), domainMigrationId, pageLimit, ) @@ -169,14 +170,14 @@ class ScanEventStoreTest extends StoreTestBase with HasExecutionContext with Spl // Fetch with cursor events2 <- fetchEvents( ctx.eventStore, - Some((domainMigrationId, recordTs1.minusSeconds(1))), + Some(TimestampWithMigrationId(recordTs1.minusSeconds(1), domainMigrationId)), domainMigrationId, pageLimit, ) // Fetch after recordTs1 events3 <- fetchEvents( ctx.eventStore, - Some((domainMigrationId, recordTs1)), + Some(TimestampWithMigrationId(recordTs1, domainMigrationId)), domainMigrationId, pageLimit, ) @@ -218,14 +219,14 @@ class ScanEventStoreTest extends StoreTestBase with HasExecutionContext with Spl // Fetch with cursor events2 <- fetchEvents( ctx.eventStore, - Some((domainMigrationId, recordTs1.minusSeconds(1))), + Some(TimestampWithMigrationId(recordTs1.minusSeconds(1), domainMigrationId)), domainMigrationId, pageLimit, ) // Fetch after recordTs1 events3 <- fetchEvents( ctx.eventStore, - Some((domainMigrationId, recordTs1)), + Some(TimestampWithMigrationId(recordTs1, domainMigrationId)), domainMigrationId, pageLimit, ) @@ -267,12 +268,17 @@ class ScanEventStoreTest extends StoreTestBase with HasExecutionContext with Spl events <- fetchEvents(ctx1.eventStore, None, mig1, pageLimit) events2 <- fetchEvents( ctx1.eventStore, - Some((mig0, recordTs1.minusSeconds(1))), + Some(TimestampWithMigrationId(recordTs1.minusSeconds(1), mig0)), mig1, pageLimit, ) // after recordTs1 - events3 <- fetchEvents(ctx1.eventStore, Some((mig0, recordTs1)), mig1, pageLimit) + events3 <- fetchEvents( + ctx1.eventStore, + Some(TimestampWithMigrationId(recordTs1, mig0)), + mig1, + pageLimit, + ) // Fetch by id works across migrationIds e1 <- ctx1.eventStore.getEventByUpdateId(updateId1, domainMigrationId) e2 <- ctx1.eventStore.getEventByUpdateId(updateId2, domainMigrationId) @@ -429,14 +435,14 @@ class ScanEventStoreTest extends StoreTestBase with HasExecutionContext with Spl // Fetch with cursor events2 <- fetchEvents( ctx.eventStore, - Some((domainMigrationId, recordTs1.minusSeconds(1))), + Some(TimestampWithMigrationId(recordTs1.minusSeconds(1), domainMigrationId)), domainMigrationId, pageLimit, ) // Fetch after latest verdict events3 <- fetchEvents( ctx.eventStore, - Some((domainMigrationId, recordTs2)), + Some(TimestampWithMigrationId(recordTs2, domainMigrationId)), domainMigrationId, pageLimit, ) @@ -474,14 +480,14 @@ class ScanEventStoreTest extends StoreTestBase with HasExecutionContext with Spl // Fetch with cursor events2 <- fetchEvents( ctx.eventStore, - Some((domainMigrationId, recordTs1.minusSeconds(1))), + Some(TimestampWithMigrationId(recordTs1.minusSeconds(1), domainMigrationId)), domainMigrationId, pageLimit, ) // Fetch after latest assignment events3 <- fetchEvents( ctx.eventStore, - Some((domainMigrationId, recordTs2)), + Some(TimestampWithMigrationId(recordTs2, domainMigrationId)), domainMigrationId, pageLimit, ) @@ -518,14 +524,14 @@ class ScanEventStoreTest extends StoreTestBase with HasExecutionContext with Spl // Fetch with cursor events2 <- fetchEvents( ctx.eventStore, - Some((domainMigrationId, recordTs1.minusSeconds(1))), + Some(TimestampWithMigrationId(recordTs1.minusSeconds(1), domainMigrationId)), domainMigrationId, pageLimit, ) // Fetch after latest verdict events3 <- fetchEvents( ctx.eventStore, - Some((domainMigrationId, recordTs2)), + Some(TimestampWithMigrationId(recordTs2, domainMigrationId)), domainMigrationId, pageLimit, ) @@ -564,14 +570,14 @@ class ScanEventStoreTest extends StoreTestBase with HasExecutionContext with Spl // Fetch with cursor events2 <- fetchEvents( ctx.eventStore, - Some((domainMigrationId, recordTs1.minusSeconds(1))), + Some(TimestampWithMigrationId(recordTs1.minusSeconds(1), domainMigrationId)), domainMigrationId, pageLimit, ) // Fetch after latest unassignment events3 <- fetchEvents( ctx.eventStore, - Some((domainMigrationId, recordTs2)), + Some(TimestampWithMigrationId(recordTs2, domainMigrationId)), domainMigrationId, pageLimit, ) @@ -813,7 +819,7 @@ class ScanEventStoreTest extends StoreTestBase with HasExecutionContext with Spl { val allow = ScanEventStore.allowF( - afterO = Some((mig0, recordTs1)), + afterO = Some(TimestampWithMigrationId(recordTs1, mig0)), currentMigrationId = mig1, currentMigrationCap = capMin, ) @@ -824,7 +830,7 @@ class ScanEventStoreTest extends StoreTestBase with HasExecutionContext with Spl { val allow = ScanEventStore.allowF( - afterO = Some((mig0, recordTs1)), + afterO = Some(TimestampWithMigrationId(recordTs1, mig0)), currentMigrationId = mig1, currentMigrationCap = cap3, ) @@ -838,7 +844,7 @@ class ScanEventStoreTest extends StoreTestBase with HasExecutionContext with Spl { val allow = ScanEventStore.allowF( - afterO = Some((mig1, recordTs2)), + afterO = Some(TimestampWithMigrationId(recordTs2, mig1)), currentMigrationId = mig1, currentMigrationCap = cap3, ) @@ -849,7 +855,7 @@ class ScanEventStoreTest extends StoreTestBase with HasExecutionContext with Spl { val allow = ScanEventStore.allowF( - afterO = Some((mig2, recordTs2)), + afterO = Some(TimestampWithMigrationId(recordTs2, mig2)), currentMigrationId = mig2, currentMigrationCap = cap2, ) @@ -1042,7 +1048,7 @@ class ScanEventStoreTest extends StoreTestBase with HasExecutionContext with Spl private def fetchEvents( es: ScanEventStore, - afterO: Option[(Long, CantonTimestamp)], + afterO: Option[TimestampWithMigrationId], currentMigrationId: Long, limit: PageLimit, ): Future[Seq[ScanEventStore#Event]] = { diff --git a/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/AcsSnapshotBulkStorageCommitFromStagingTest.scala b/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/AcsSnapshotBulkStorageCommitFromStagingTest.scala index 5159c40f1b..2b3b0a314b 100644 --- a/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/AcsSnapshotBulkStorageCommitFromStagingTest.scala +++ b/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/AcsSnapshotBulkStorageCommitFromStagingTest.scala @@ -106,6 +106,7 @@ class AcsSnapshotBulkStorageCommitFromStagingTest reader, appConfig, null, // not used when bft reads are disabled + _ => (), loggerFactory, ) val commitService = { diff --git a/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/AcsSnapshotBulkStorageWriterFromDbTest.scala b/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/AcsSnapshotBulkStorageWriterFromDbTest.scala index a79ad57c8d..d5768f5968 100644 --- a/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/AcsSnapshotBulkStorageWriterFromDbTest.scala +++ b/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/AcsSnapshotBulkStorageWriterFromDbTest.scala @@ -162,7 +162,13 @@ class AcsSnapshotBulkStorageWriterFromDbTest .get(MetricsContext.Empty) .value .markers - .get(MetricsContext("object_type" -> "ACS_snapshots", "encoding" -> encoding.key)) + .get( + MetricsContext( + "object_type" -> "ACS_snapshots", + "encoding" -> encoding.key, + "bucket" -> "staging", + ) + ) .value .get() numObjectsFromMetric shouldBe expectedDigests.length @@ -349,7 +355,7 @@ class AcsSnapshotBulkStorageWriterFromDbTest store.queryAcsSnapshot( anyLong, any[CantonTimestamp], - any[Option[Long]], + any[Option[AcsSnapshotStore.QueryAcsSnapshotPaginationToken]], any[Limit], any[Seq[PartyId]], any[Seq[PackageQualifiedName]], @@ -358,14 +364,22 @@ class AcsSnapshotBulkStorageWriterFromDbTest ( migration: Long, timestamp: CantonTimestamp, - after: Option[Long], + after: Option[AcsSnapshotStore.QueryAcsSnapshotPaginationToken], limit: Limit, _: Seq[PartyId], _: Seq[PackageQualifiedName], ) => if (snapshots.contains(timestamp)) { Future { - val remaining = snapshotSize - after.getOrElse(0L) + val afterAsLong = after match { + case Some( + AcsSnapshotStore.QueryAcsSnapshotPaginationToken + .RowIdQueryAcsSnapshotPaginationToken(value) + ) => + value + case None => 0L + } + val remaining = snapshotSize - afterAsLong val numElems = math.min(limit.limit.toLong, remaining) val result = QueryAcsSnapshotResult( migration, @@ -373,7 +387,7 @@ class AcsSnapshotBulkStorageWriterFromDbTest Vector .range(0, numElems) .map(i => { - val idx = i + after.getOrElse(0L) + val idx = i + afterAsLong val amt = amulet( partyId, BigDecimal(idx), @@ -388,7 +402,12 @@ class AcsSnapshotBulkStorageWriterFromDbTest toCreatedEvent(amt), ) }), - if (numElems < remaining) Some(after.getOrElse(0L) + numElems) else None, + if (numElems < remaining) + Some( + AcsSnapshotStore.QueryAcsSnapshotPaginationToken + .RowIdQueryAcsSnapshotPaginationToken(afterAsLong + numElems) + ) + else None, ) result } diff --git a/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/BulkStorageCommitFromStagingTest.scala b/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/BulkStorageCommitFromStagingTest.scala index 7391b45019..639d6ac845 100644 --- a/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/BulkStorageCommitFromStagingTest.scala +++ b/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/BulkStorageCommitFromStagingTest.scala @@ -188,12 +188,57 @@ class BulkStorageCommitFromStagingTest logEntries => forAtLeast(1, logEntries)( _.message should include( - "All objects are known to the BFT peers, but the checksums do not match" + "Checksums do not match for objects" ) ), ) } + "ignore digest mismatches for objects listed in debugObjectsToNotCommit and not copy them to the committed bucket" in { + val (stagingS3Connection, committedS3Connection, objsWithDigests) = setupTest + + val ignoredObject = objsWithDigests(1) + + val mockScanConnections = new MockScanConnections(objsWithDigests) + // all peers disagree with us on the digest of the ignored object only + Seq.range(0, 7).foreach(i => mockScanConnections.scanDisagreesOnDigest(i, 1)) + + val flow = newCopyFlow( + stagingS3Connection, + committedS3Connection, + objsWithDigests, + mockScanConnections, + appConfig.copy(debugObjectsToNotCommit = Seq(ignoredObject.key)), + ) + + loggerFactory.assertLogsSeq(SuppressionRule.LevelAndAbove(Level.ERROR))( + { + triggerCopyFlowAndAssertCompletion(flow) + }, + logEntries => + forAll(logEntries)( + _.message should include("Checksums do not match for objects") + ), + ) + + val expectedCommittedObjects = objsWithDigests.filterNot(_.key == ignoredObject.key) + + clue("All objects have been deleted from staging") { + stagingS3Connection.listObjects.futureValue.contents().asScala shouldBe empty + } + clue("Only the non-ignored objects have been copied to the committed bucket") { + committedS3Connection.listObjects.futureValue + .contents() + .asScala + .map(_.key()) should contain theSameElementsAs expectedCommittedObjects.map(_.key) + } + clue("Checksums of objects in committed S3 bucket match the expected digests") { + committedS3Connection + .getChecksums(expectedCommittedObjects.map(_.key)) + .futureValue should contain theSameElementsAs expectedCommittedObjects + } + } + class MockScanConnections( objsWithDigests: Seq[ObjectKeyAndChecksum] ) { @@ -292,12 +337,13 @@ class BulkStorageCommitFromStagingTest committedS3Connection: S3BucketConnectionForUnitTests, objsWithDigests: Seq[ObjectKeyAndChecksum], mockScanConnections: MockScanConnections, + config: BulkStorageConfig = appConfig, ) = { new BulkStorageCommitFromStaging[String]( stagingS3Connection, committedS3Connection, _ => Future.successful(objsWithDigests), - appConfig, + config, mockScanConnections.peerBftConnection, loggerFactory, ).getFlow diff --git a/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/S3UploadTest.scala b/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/S3UploadTest.scala index 61a0883a79..586ef6f0fa 100644 --- a/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/S3UploadTest.scala +++ b/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/S3UploadTest.scala @@ -6,15 +6,21 @@ package org.lfdecentralizedtrust.splice.scan.store.bulk import org.apache.pekko.stream.scaladsl.Keep import org.apache.pekko.stream.testkit.scaladsl.{TestSink, TestSource} import org.apache.pekko.util.ByteString +import org.lfdecentralizedtrust.splice.config.S3Config import org.lfdecentralizedtrust.splice.store.{HasS3Mock, S3BucketConnection, StoreTestBase} +import com.digitalasset.canton.logging.NamedLoggerFactory import scala.util.Random import scala.concurrent.duration.* +import scala.concurrent.{ExecutionContext, Future, Promise} import scala.jdk.CollectionConverters.* import java.nio.ByteBuffer +import java.util.concurrent.atomic.AtomicInteger class S3UploadTest extends StoreTestBase with HasS3Mock { + private val emptyDigest = "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" + "S3 multipart uploads" should { "work" in { @@ -34,6 +40,27 @@ class S3UploadTest extends StoreTestBase with HasS3Mock { new String(content.toArray, "UTF-8") shouldBe "helloworld" } } + + "not corrupt the checksum if finish() is called more than once" in { + val expectedContent = "idempotency test" + val bucketConnection = new S3BucketConnectionForUnitTests(s3ConfigMock(), loggerFactory) + val o = bucketConnection.newAppendWriteObject("finish-twice") + val part = ByteBuffer.wrap(expectedContent.getBytes("UTF-8")) + + o.prepareUploadNext(part) + for { + _ <- o.upload(1, part) + _ <- o.finish() + checksumAfterFirstFinish <- bucketConnection.getChecksums(Seq("finish-twice")) + _ <- o.finish() + checksumAfterSecondFinish <- bucketConnection.getChecksums(Seq("finish-twice")) + content <- bucketConnection.readFullObject("finish-twice") + } yield { + checksumAfterFirstFinish.map(_.checksum) should not contain emptyDigest + checksumAfterSecondFinish shouldBe checksumAfterFirstFinish + new String(content.toArray, "UTF-8") shouldBe expectedContent + } + } } "GroupedWeightS3Object" should { @@ -116,6 +143,77 @@ class S3UploadTest extends StoreTestBase with HasS3Mock { sub.expectError() succeed } + + "not finish an object twice when upstream completes while the object is being finished" in { + // Regression test for the race that produced correct object content with a wrong checksum: + // an object that is done by size starts being finished from uploadCallback; `state` is only + // advanced later, in the async finishCallback. Upstream completion is delivered eagerly + // (independently of demand), so onUpstreamFinish could land in that window and call finish() + // a second time on the very same object. + val bucketConnection = new GatedFinishS3Connection(s3ConfigMock(), loggerFactory) + + val (pub, sub) = TestSource + .probe[ByteString] + .via( + GroupedWeightS3ObjectFlow( + bucketConnection, + getObjectKey = i => s"race_$i", + maxObjectSize = 10L, + maxParallelPartUploads = 2, + loggerFactory, + ) + ) + .toMat(TestSink.probe[String])(Keep.both) + .run() + + sub.request(5) + // Exactly hits maxObjectSize, so the object is done by size and finish() is started + // from uploadCallback as soon as the single part upload completes. + pub.sendNext(ByteString(Random.nextBytes(10))) + + // Wait until the flow is blocked inside finish() + eventually() { + bucketConnection.finishCount.get() shouldBe 1 + } + + // Complete upstream while finish() is still in flight + pub.sendComplete() + always(durationOfSuccess = 2.seconds) { + bucketConnection.finishCount.get() shouldBe 1 + } + + bucketConnection.releaseFinish() + sub.expectNext(20.seconds) shouldBe "race_0" + sub.expectComplete() + + val checksums = bucketConnection.getChecksums(Seq("race_0")).futureValue + checksums should have size 1 + checksums.map(_.checksum) should not contain emptyDigest + succeed + } + } + + /** An S3 connection whose `finish()` blocks until [[releaseFinish]] is called, and which counts + * how many times `finish()` was invoked. + */ + private class GatedFinishS3Connection( + s3Config: S3Config, + loggerFactory: NamedLoggerFactory, + ) extends S3BucketConnectionForUnitTests(s3Config, loggerFactory) { + val finishCount = new AtomicInteger(0) + private val gate = Promise[Unit]() + + def releaseFinish(): Unit = { val _ = gate.trySuccess(()) } + + override def newAppendWriteObject( + key: String + )(implicit ec: ExecutionContext): AppendWriteObject = + new AppendWriteObjectForUnitTests(key) { + override def finish(): Future[Unit] = { + val _ = finishCount.incrementAndGet() + gate.future.flatMap(_ => super.finish()) + } + } } "S3BucketConnection" should { diff --git a/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/UpdateHistoryBulkStorageTest.scala b/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/UpdateHistoryBulkStorageTest.scala index eedad265c6..6a9eed5319 100644 --- a/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/UpdateHistoryBulkStorageTest.scala +++ b/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/scan/store/bulk/UpdateHistoryBulkStorageTest.scala @@ -135,7 +135,13 @@ class UpdateHistoryBulkStorageTest .get(MetricsContext.Empty) .value .markers - .get(MetricsContext("object_type" -> "updates", "encoding" -> encoding.key)) + .get( + MetricsContext( + "object_type" -> "updates", + "encoding" -> encoding.key, + "bucket" -> "staging", + ) + ) .value .get() numObjectsFromMetric(ScanStorageConfig.Encoding.CompactJson) shouldBe 2 @@ -580,17 +586,15 @@ class UpdateHistoryBulkStorageTest val store = mock[UpdateHistory] when( store.getUpdatesWithoutImportUpdates( - any[Option[(Long, CantonTimestamp)]], + any[Option[TimestampWithMigrationId]], any[Limit], )(any[TraceContext]) ).thenAnswer { ( - afterO: Option[(Long, CantonTimestamp)], + afterO: Option[TimestampWithMigrationId], limit: Limit, ) => - val after = afterO - .map(a => TimestampWithMigrationId(a._2, a._1)) - .getOrElse(TimestampWithMigrationId(CantonTimestamp.MinValue, 0L)) + val after = afterO.getOrElse(TimestampWithMigrationId(CantonTimestamp.MinValue, 0L)) Future.successful( data .filter(update => diff --git a/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/store/db/AcsSnapshotStoreTest.scala b/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/store/db/AcsSnapshotStoreTest.scala index 4c27a6c133..c3574b6cfd 100644 --- a/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/store/db/AcsSnapshotStoreTest.scala +++ b/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/store/db/AcsSnapshotStoreTest.scala @@ -432,7 +432,7 @@ class AcsSnapshotStoreTest def queryRecursive( store: AcsSnapshotStore, - after: Option[Long], + after: Option[AcsSnapshotStore.QueryAcsSnapshotPaginationToken], acc: Vector[String], partyIds: Seq[PartyId], templates: Seq[PackageQualifiedName], diff --git a/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/store/db/DbScanRewardsReferenceStoreTest.scala b/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/store/db/DbScanRewardsReferenceStoreTest.scala index 686f647f03..0e2ad85086 100644 --- a/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/store/db/DbScanRewardsReferenceStoreTest.scala +++ b/apps/scan/src/test/scala/org/lfdecentralizedtrust/splice/store/db/DbScanRewardsReferenceStoreTest.scala @@ -24,7 +24,14 @@ import org.lfdecentralizedtrust.splice.environment.ledger.api.TreeUpdateOrOffset import org.lfdecentralizedtrust.splice.environment.{DarResources, RetryProvider} import org.lfdecentralizedtrust.splice.scan.store.ScanRewardsReferenceStore import org.lfdecentralizedtrust.splice.scan.store.db.DbScanRewardsReferenceStore -import org.lfdecentralizedtrust.splice.store.{HardLimit, Limit, PageLimit, StoreTestBase, TcsStore} +import org.lfdecentralizedtrust.splice.store.{ + HardLimit, + Limit, + PageLimit, + StoreTestBase, + TcsStore, + TimestampWithMigrationId, +} import org.lfdecentralizedtrust.splice.util.{ResourceTemplateDecoder, TemplateJsonDecoder} import slick.jdbc.JdbcProfile @@ -479,10 +486,16 @@ class DbScanRewardsReferenceStoreTest result.get(ts(275)) shouldBe None // round4.opensAt before earliest archived_at result.get(ts(350)) shouldBe None // round4.opensAt before earliest archived_at result.get(ts(375)) shouldBe None // gap: round4 archived, round5 not yet open - result(ts(400)) shouldBe (5L, ts(400)) + result(ts(400)) shouldBe TimestampWithMigrationId(ts(400), 5L) result.get(ts(401)) shouldBe None // 401 was not present in request - result(ts(450)) shouldBe (5L, ts(400)) // round5 open, round6 not yet open - result(ts(550)) shouldBe (5L, ts(400)) // both open, lowest round selected + result(ts(450)) shouldBe TimestampWithMigrationId( + ts(400), + 5L, + ) // round5 open, round6 not yet open + result(ts(550)) shouldBe TimestampWithMigrationId( + ts(400), + 5L, + ) // both open, lowest round selected } } } diff --git a/apps/sv/frontend/index.html b/apps/sv/frontend/index.html index c2f95b8233..32252600a0 100644 --- a/apps/sv/frontend/index.html +++ b/apps/sv/frontend/index.html @@ -7,7 +7,7 @@ diff --git a/apps/sv/frontend/src/__tests__/components/copyable-identifier.test.tsx b/apps/sv/frontend/src/__tests__/components/copyable-identifier.test.tsx index 0f2d6ec304..90a415352d 100644 --- a/apps/sv/frontend/src/__tests__/components/copyable-identifier.test.tsx +++ b/apps/sv/frontend/src/__tests__/components/copyable-identifier.test.tsx @@ -27,26 +27,80 @@ describe('CopyableIdentifier', () => { expect(screen.getByTestId('contract-id-scroll')).toHaveStyle({ overflowX: 'auto' }); }); - test('shows a scroll track below the identifier when content overflows', async () => { + test('keeps the copy button adjacent to short identifiers', () => { + render( +
+ +
+ ); + + expect(screen.getByTestId('short-id')).toHaveStyle({ + display: 'inline-flex', + width: 'fit-content', + }); + }); + + test('compact scroll keeps #1785 scrolling at the Figma width', async () => { render( - + ); const scroll = screen.getByTestId('contract-id-scroll'); + expect(scroll).toHaveStyle({ overflowX: 'auto', maxWidth: '270px' }); + expect(screen.getByTestId('contract-id-value')).toHaveTextContent(LONG_CONTRACT_ID); + Object.defineProperty(scroll, 'scrollWidth', { configurable: true, value: 400 }); Object.defineProperty(scroll, 'clientWidth', { configurable: true, value: 100 }); + Object.defineProperty(scroll, 'scrollLeft', { configurable: true, value: 0 }); fireEvent.scroll(scroll); await waitFor(() => { + expect(screen.queryByTestId('contract-id-ellipsis-cue')).not.toBeInTheDocument(); expect(screen.getByTestId('contract-id-scroll-track')).toBeInTheDocument(); }); + }); + + test('fullWidth fills the parent and keeps scrolling', () => { + render( +
+ +
+ ); + + expect(screen.getByTestId('contract-id')).toHaveStyle({ width: '100%', display: 'flex' }); + expect(screen.getByTestId('contract-id-scroll')).toHaveStyle({ overflowX: 'auto' }); + expect(screen.getByTestId('contract-id-value')).toHaveTextContent(LONG_CONTRACT_ID); + }); - expect(screen.getByTestId('contract-id-scroll-track')).toHaveStyle({ - opacity: '0', - height: '0px', + test('trims long identifiers to the Figma ellipsis width without a narrow parent', () => { + render( + + ); + + expect(screen.getByTestId('contract-id-value')).toHaveTextContent(LONG_CONTRACT_ID); + expect(screen.getByTestId('contract-id-value')).toHaveAttribute('title', LONG_CONTRACT_ID); + expect(screen.getByTestId('contract-id-ellipsis')).toHaveStyle({ + overflow: 'hidden', + maxWidth: '270px', }); + expect(screen.getByTestId('contract-id-value')).toHaveStyle({ textOverflow: 'ellipsis' }); }); }); @@ -60,6 +114,26 @@ describe('MemberIdentifier', () => { expect(screen.getByTestId('member-value')).not.toHaveTextContent('...'); expect(screen.getByTestId('member-scroll')).toHaveStyle({ overflowX: 'auto' }); }); + + test('supports ellipsis overflow for compact layouts', () => { + render( + + + + ); + + expect(screen.getByTestId('member-value')).toHaveTextContent(LONG_PARTY_ID); + expect(screen.getByTestId('member-value')).toHaveAttribute('title', LONG_PARTY_ID); + expect(screen.getByTestId('member-ellipsis')).toHaveStyle({ overflow: 'hidden' }); + expect(screen.getByTestId('member-value')).toHaveStyle({ textOverflow: 'ellipsis' }); + expect(screen.queryByTestId('member-scroll')).not.toBeInTheDocument(); + }); }); describe('common PartyId', () => { diff --git a/apps/sv/frontend/src/__tests__/governance/create-proposal.test.tsx b/apps/sv/frontend/src/__tests__/governance/create-proposal.test.tsx index d85fb78cdc..14c8db40f3 100644 --- a/apps/sv/frontend/src/__tests__/governance/create-proposal.test.tsx +++ b/apps/sv/frontend/src/__tests__/governance/create-proposal.test.tsx @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { render, screen, waitFor } from '@testing-library/react'; -import { describe, expect, test } from 'vitest'; +import { beforeAll, describe, expect, test } from 'vitest'; import { MemoryRouter } from 'react-router'; import { ThemeProvider } from '@emotion/react'; import { theme } from '../../../../../common/frontend/lib/theme'; @@ -50,6 +50,12 @@ async function checkActionSelection(actionName: string, actionValue: string, tes expect(actionInput.textContent).toBe(action!.name); } +// The SV app's /v1/dso endpoint requires authentication, so log in before rendering +// (matches the UserProvider's session-restore path for test auth). +beforeAll(() => { + window.sessionStorage.setItem('canton.network.wallet.userid', 'sv1'); +}); + describe('Create Proposal', () => { test('Does not render the form while dsoInfo is pending, then lands on +7d default', async () => { let releaseDso!: () => void; @@ -58,7 +64,7 @@ describe('Create Proposal', () => { }); server.use( - http.get(`${svUrl}/v0/dso`, async () => { + http.get(`${svUrl}/v1/dso`, async () => { await dsoReady; return HttpResponse.json(dsoInfo); }) diff --git a/apps/sv/frontend/src/__tests__/governance/forms/create-unallocated-unclaimed-activity-form.test.tsx b/apps/sv/frontend/src/__tests__/governance/forms/create-unallocated-unclaimed-activity-form.test.tsx index 5c13445a01..67858a367d 100644 --- a/apps/sv/frontend/src/__tests__/governance/forms/create-unallocated-unclaimed-activity-form.test.tsx +++ b/apps/sv/frontend/src/__tests__/governance/forms/create-unallocated-unclaimed-activity-form.test.tsx @@ -10,7 +10,11 @@ import { describe, expect, test } from 'vitest'; import App from '../../../App'; import { CreateUnallocatedUnclaimedActivityRecordForm } from '../../../components/forms/CreateUnallocatedUnclaimedActivityRecordForm'; import { SvConfigProvider } from '../../../utils'; -import { PROPOSAL_SUMMARY_SUBTITLE, PROPOSAL_SUMMARY_TITLE } from '../../../utils/constants'; +import { + CREATE_PROPOSAL_LABEL_PROPOSAL_TYPE, + PROPOSAL_REVIEW_TITLE, + PROPOSAL_SUMMARY_SUBTITLE, +} from '../../../utils/constants'; import { Wrapper } from '../../helpers'; import { svPartyId } from '../../mocks/constants'; import { server, svUrl } from '../../setup/setup'; @@ -47,7 +51,7 @@ describe('Create Unallocated Unclaimed Activity Record Form', () => { expect( screen.getByTestId('create-unallocated-unclaimed-activity-record-form') ).toBeInTheDocument(); - expect(screen.getByText('Proposal type')).toBeInTheDocument(); + expect(screen.getByText(CREATE_PROPOSAL_LABEL_PROPOSAL_TYPE)).toBeInTheDocument(); const actionInput = screen.getByTestId('create-unallocated-unclaimed-activity-record-action'); expect(actionInput).toBeInTheDocument(); @@ -368,7 +372,7 @@ describe('Create Unallocated Unclaimed Activity Record Form', () => { await user.click(submitButton); - expect(screen.getByText(PROPOSAL_SUMMARY_TITLE)).toBeInTheDocument(); + expect(screen.getByText(PROPOSAL_REVIEW_TITLE)).toBeInTheDocument(); }); test('should show error on form if submission fails', async () => { diff --git a/apps/sv/frontend/src/__tests__/governance/forms/grant-revoke-featured-app-form.test.tsx b/apps/sv/frontend/src/__tests__/governance/forms/grant-revoke-featured-app-form.test.tsx index 1566a5ff9e..aaca03db31 100644 --- a/apps/sv/frontend/src/__tests__/governance/forms/grant-revoke-featured-app-form.test.tsx +++ b/apps/sv/frontend/src/__tests__/governance/forms/grant-revoke-featured-app-form.test.tsx @@ -13,7 +13,12 @@ import dayjs from 'dayjs'; import { GrantRevokeFeaturedAppForm } from '../../../components/forms/GrantRevokeFeaturedAppForm'; import { server, svUrl } from '../../setup/setup'; import { http, HttpResponse } from 'msw'; -import { PROPOSAL_SUMMARY_SUBTITLE, PROPOSAL_SUMMARY_TITLE } from '../../../utils/constants'; +import { + CREATE_PROPOSAL_LABEL_PROPOSAL_TYPE, + CREATE_PROPOSAL_LABEL_PROVIDER_PARTY_ID, + PROPOSAL_REVIEW_TITLE, + PROPOSAL_SUMMARY_SUBTITLE, +} from '../../../utils/constants'; describe('SV user can', () => { test('login and see the SV party ID', async () => { @@ -45,7 +50,7 @@ describe('Grant Featured App Form', () => { ); expect(screen.getByTestId('grant-featured-app-form')).toBeInTheDocument(); - expect(screen.getByText('Proposal type')).toBeInTheDocument(); + expect(screen.getByText(CREATE_PROPOSAL_LABEL_PROPOSAL_TYPE)).toBeInTheDocument(); const actionInput = screen.getByTestId('grant-featured-app-action'); expect(actionInput).toBeInTheDocument(); @@ -69,7 +74,7 @@ describe('Grant Featured App Form', () => { const providerInput = screen.getByTestId('grant-featured-app-idValue-title'); expect(providerInput).toBeInTheDocument(); - expect(providerInput.textContent).toBe('Provider Party ID'); + expect(providerInput.textContent).toBe(CREATE_PROPOSAL_LABEL_PROVIDER_PARTY_ID); expect(screen.getByText('Review Proposal')).toBeInTheDocument(); }); @@ -212,7 +217,7 @@ describe('Grant Featured App Form', () => { await user.click(submitButton); - expect(screen.getByText(PROPOSAL_SUMMARY_TITLE)).toBeInTheDocument(); + expect(screen.getByText(PROPOSAL_REVIEW_TITLE)).toBeInTheDocument(); }); test('activity weight is optional and is sent to backend as null when left blank', async () => { @@ -391,7 +396,7 @@ describe('Revoke Featured App Form', () => { ); expect(screen.getByTestId('revoke-featured-app-form')).toBeInTheDocument(); - expect(screen.getByText('Proposal type')).toBeInTheDocument(); + expect(screen.getByText(CREATE_PROPOSAL_LABEL_PROPOSAL_TYPE)).toBeInTheDocument(); const actionInput = screen.getByTestId('revoke-featured-app-action'); expect(actionInput).toBeInTheDocument(); @@ -411,7 +416,7 @@ describe('Revoke Featured App Form', () => { const partyIdTitle = screen.getByTestId('revoke-featured-app-partyId-title'); expect(partyIdTitle).toBeInTheDocument(); - expect(partyIdTitle.textContent).toBe('Provider Party ID'); + expect(partyIdTitle.textContent).toBe(CREATE_PROPOSAL_LABEL_PROVIDER_PARTY_ID); const rightCidDropdown = screen.getByTestId('revoke-featured-app-rightCid-dropdown'); expect(rightCidDropdown).toBeInTheDocument(); @@ -544,7 +549,7 @@ describe('Revoke Featured App Form', () => { await user.click(submitButton); - expect(screen.getByText(PROPOSAL_SUMMARY_TITLE)).toBeInTheDocument(); + expect(screen.getByText(PROPOSAL_REVIEW_TITLE)).toBeInTheDocument(); expect(screen.getByTestId('revokeProviderPartyId-title').textContent).toBe('Provider Party ID'); expect(screen.getByTestId('revokeProviderPartyId-field').textContent).toBe( 'a-party-id::1014912492' diff --git a/apps/sv/frontend/src/__tests__/governance/forms/offboard-sv-form.test.tsx b/apps/sv/frontend/src/__tests__/governance/forms/offboard-sv-form.test.tsx index 3d95c438be..48723ab78f 100644 --- a/apps/sv/frontend/src/__tests__/governance/forms/offboard-sv-form.test.tsx +++ b/apps/sv/frontend/src/__tests__/governance/forms/offboard-sv-form.test.tsx @@ -13,7 +13,13 @@ import dayjs from 'dayjs'; import { OffboardSvForm } from '../../../components/forms/OffboardSvForm'; import { server, svUrl } from '../../setup/setup'; import { http, HttpResponse } from 'msw'; -import { PROPOSAL_SUMMARY_SUBTITLE, PROPOSAL_SUMMARY_TITLE } from '../../../utils/constants'; +import { + CREATE_PROPOSAL_LABEL_PROPOSAL_TYPE, + PROPOSAL_REVIEW_TITLE, + PROPOSAL_SUMMARY_PLACEHOLDER, + PROPOSAL_SUMMARY_SUBTITLE, + URL_PLACEHOLDER, +} from '../../../utils/constants'; describe('SV user can', () => { test('login and see the SV party ID', async () => { @@ -45,7 +51,7 @@ describe('Offboard SV Form', () => { ); expect(screen.getByTestId('offboard-sv-form')).toBeInTheDocument(); - expect(screen.getByText('Proposal type')).toBeInTheDocument(); + expect(screen.getByText(CREATE_PROPOSAL_LABEL_PROPOSAL_TYPE)).toBeInTheDocument(); const actionInput = screen.getByTestId('offboard-sv-action'); expect(actionInput).toBeInTheDocument(); @@ -54,6 +60,7 @@ describe('Offboard SV Form', () => { const summaryInput = screen.getByTestId('offboard-sv-summary'); expect(summaryInput).toBeInTheDocument(); expect(summaryInput.getAttribute('value')).toBeNull(); + expect(summaryInput.getAttribute('placeholder')).toBe(PROPOSAL_SUMMARY_PLACEHOLDER); const summarySubtitle = screen.getByTestId('offboard-sv-summary-subtitle'); expect(summarySubtitle).toBeInTheDocument(); @@ -62,10 +69,12 @@ describe('Offboard SV Form', () => { const urlInput = screen.getByTestId('offboard-sv-url'); expect(urlInput).toBeInTheDocument(); expect(urlInput.getAttribute('value')).toBe(''); + expect(urlInput).toHaveAttribute('placeholder', URL_PLACEHOLDER); const memberInput = screen.getByTestId('offboard-sv-member-dropdown'); expect(memberInput).toBeInTheDocument(); expect(memberInput.getAttribute('value')).toBe(''); + expect(screen.getByText('Select a member')).toBeInTheDocument(); expect(screen.getByText('Review Proposal')).toBeInTheDocument(); }); @@ -231,7 +240,7 @@ describe('Offboard SV Form', () => { await user.click(submitButton); - expect(screen.getByText(PROPOSAL_SUMMARY_TITLE)).toBeInTheDocument(); + expect(screen.getByText(PROPOSAL_REVIEW_TITLE)).toBeInTheDocument(); }); test('should show error on form if submission fails', async () => { diff --git a/apps/sv/frontend/src/__tests__/governance/forms/set-amulet-rules-form.test.tsx b/apps/sv/frontend/src/__tests__/governance/forms/set-amulet-rules-form.test.tsx index d12a78ac96..a4cc181b8f 100644 --- a/apps/sv/frontend/src/__tests__/governance/forms/set-amulet-rules-form.test.tsx +++ b/apps/sv/frontend/src/__tests__/governance/forms/set-amulet-rules-form.test.tsx @@ -13,7 +13,11 @@ import { SetAmuletConfigRulesForm } from '../../../components/forms/SetAmuletCon import dayjs from 'dayjs'; import { dateTimeFormatISO } from '@canton-network/splice-common-frontend-utils'; import { server, svUrl } from '../../setup/setup'; -import { PROPOSAL_SUMMARY_SUBTITLE, PROPOSAL_SUMMARY_TITLE } from '../../../utils/constants'; +import { + CREATE_PROPOSAL_LABEL_PROPOSAL_TYPE, + PROPOSAL_REVIEW_TITLE, + PROPOSAL_SUMMARY_SUBTITLE, +} from '../../../utils/constants'; describe('SV user can', () => { test('login and see the SV party ID', async () => { @@ -45,7 +49,7 @@ describe('Set Amulet Config Rules Form', () => { ); expect(screen.getByTestId('set-amulet-config-rules-form')).toBeInTheDocument(); - expect(screen.getByText('Proposal type')).toBeInTheDocument(); + expect(screen.getByText(CREATE_PROPOSAL_LABEL_PROPOSAL_TYPE)).toBeInTheDocument(); const actionInput = screen.getByTestId('set-amulet-config-rules-action'); expect(actionInput).toBeInTheDocument(); @@ -122,7 +126,7 @@ describe('Set Amulet Config Rules Form', () => { await user.click(actionInput); // using this to trigger the onBlur event which triggers the validation - expect(submitButton.getAttribute('disabled')).toBeNull(); + await waitFor(() => expect(submitButton.getAttribute('disabled')).toBeNull()); }, { timeout: 10000 } ); @@ -325,7 +329,7 @@ describe('Set Amulet Config Rules Form', () => { await user.click(submitButton); - expect(screen.getByText(PROPOSAL_SUMMARY_TITLE)).toBeInTheDocument(); + expect(screen.getByText(PROPOSAL_REVIEW_TITLE)).toBeInTheDocument(); expect(screen.queryByText('JSON')).not.toBeInTheDocument(); expect(screen.getByTestId('json-diff-toggle')).toHaveTextContent('Show JSON'); }); diff --git a/apps/sv/frontend/src/__tests__/governance/forms/set-dso-rules-form.test.tsx b/apps/sv/frontend/src/__tests__/governance/forms/set-dso-rules-form.test.tsx index 2825f1d132..471a378f50 100644 --- a/apps/sv/frontend/src/__tests__/governance/forms/set-dso-rules-form.test.tsx +++ b/apps/sv/frontend/src/__tests__/governance/forms/set-dso-rules-form.test.tsx @@ -12,6 +12,12 @@ import { http, HttpResponse } from 'msw'; import { describe, expect, test } from 'vitest'; import App from '../../../App'; import { SetDsoConfigRulesForm } from '../../../components/forms/SetDsoConfigRulesForm'; +import { + CREATE_PROPOSAL_LABEL_PROPOSAL_TYPE, + DATE_TIME_PLACEHOLDER, + REASON_PLACEHOLDER, + URL_PLACEHOLDER, +} from '../../../utils/constants'; import { SvConfigProvider } from '../../../utils'; import { Wrapper } from '../../helpers'; import { svPartyId } from '../../mocks/constants'; @@ -47,7 +53,7 @@ describe('Set DSO Config Rules Form', () => { ); expect(screen.getByTestId('set-dso-config-rules-form')).toBeInTheDocument(); - expect(screen.getByText('Proposal type')).toBeInTheDocument(); + expect(screen.getByText(CREATE_PROPOSAL_LABEL_PROPOSAL_TYPE)).toBeInTheDocument(); const actionInput = screen.getByTestId('set-dso-config-rules-action'); expect(actionInput).toBeInTheDocument(); @@ -58,10 +64,21 @@ describe('Set DSO Config Rules Form', () => { const summaryInput = screen.getByTestId('set-dso-config-rules-summary'); expect(summaryInput).toBeInTheDocument(); expect(summaryInput.getAttribute('value')).not.toBeInTheDocument(); + expect(summaryInput).toHaveAttribute('placeholder', REASON_PLACEHOLDER); const urlInput = screen.getByTestId('set-dso-config-rules-url'); expect(urlInput).toBeInTheDocument(); expect(urlInput.getAttribute('value')).toBe(''); + expect(urlInput).toHaveAttribute('placeholder', URL_PLACEHOLDER); + + expect(screen.getByTestId('set-dso-config-rules-expiry-date-field')).toHaveAttribute( + 'placeholder', + DATE_TIME_PLACEHOLDER + ); + expect(screen.getByTestId('set-dso-config-rules-effective-date-field')).toHaveAttribute( + 'placeholder', + DATE_TIME_PLACEHOLDER + ); const configLabels = screen.getAllByTestId(/config-label-/); expect(configLabels.length).toBeGreaterThan(15); @@ -113,7 +130,7 @@ describe('Set DSO Config Rules Form', () => { await user.click(actionInput); // using this to trigger the onBlur event which triggers the validation - expect(submitButton.getAttribute('disabled')).not.toBeInTheDocument(); + await waitFor(() => expect(submitButton.getAttribute('disabled')).toBeNull()); }); test('expiry date must be in the future', async () => { @@ -267,7 +284,7 @@ describe('Set DSO Config Rules Form', () => { await user.click(submitButton); - expect(screen.getByText('Proposal Summary')).toBeInTheDocument(); + expect(screen.getByText('Proposal Review')).toBeInTheDocument(); expect(screen.queryByText('JSON')).not.toBeInTheDocument(); expect(screen.getByTestId('json-diff-toggle')).toHaveTextContent('Show JSON'); }); diff --git a/apps/sv/frontend/src/__tests__/governance/forms/update-sv-reward-weight-form-test.test.tsx b/apps/sv/frontend/src/__tests__/governance/forms/update-sv-reward-weight-form-test.test.tsx index a9bd4985ee..2d6892d4bb 100644 --- a/apps/sv/frontend/src/__tests__/governance/forms/update-sv-reward-weight-form-test.test.tsx +++ b/apps/sv/frontend/src/__tests__/governance/forms/update-sv-reward-weight-form-test.test.tsx @@ -13,7 +13,10 @@ import { dateTimeFormatISO } from '@canton-network/splice-common-frontend-utils' import dayjs from 'dayjs'; import { server, svUrl } from '../../setup/setup'; import { http, HttpResponse } from 'msw'; -import { PROPOSAL_SUMMARY_SUBTITLE } from '../../../utils/constants'; +import { + CREATE_PROPOSAL_LABEL_PROPOSAL_TYPE, + PROPOSAL_SUMMARY_SUBTITLE, +} from '../../../utils/constants'; describe('SV user can', () => { test('login and see the SV party ID', async () => { @@ -45,7 +48,7 @@ describe('Update Super Validator Reward Weight Form', () => { ); expect(screen.getByTestId('update-sv-reward-weight-form')).toBeInTheDocument(); - expect(screen.getByText('Proposal type')).toBeInTheDocument(); + expect(screen.getByText(CREATE_PROPOSAL_LABEL_PROPOSAL_TYPE)).toBeInTheDocument(); const actionInput = screen.getByTestId('update-sv-reward-weight-action'); expect(actionInput).toBeInTheDocument(); diff --git a/apps/sv/frontend/src/__tests__/governance/governance-page.test.tsx b/apps/sv/frontend/src/__tests__/governance/governance-page.test.tsx index fa85a645a9..931fa7d068 100644 --- a/apps/sv/frontend/src/__tests__/governance/governance-page.test.tsx +++ b/apps/sv/frontend/src/__tests__/governance/governance-page.test.tsx @@ -183,7 +183,7 @@ describe('Governance Page', () => { await user.click(viewDetailsLink); - const proposalDetails = screen.getByTestId('proposal-details-title'); + const proposalDetails = screen.getByTestId('proposal-details-proposal-details'); expect(proposalDetails).toBeInTheDocument(); }); @@ -201,7 +201,7 @@ describe('Governance Page', () => { await user.click(viewDetailsLink); - const proposalDetails = screen.getByTestId('proposal-details-title'); + const proposalDetails = screen.getByTestId('proposal-details-proposal-details'); expect(proposalDetails).toBeInTheDocument(); const action = screen.getByTestId('proposal-details-action-value'); @@ -220,6 +220,11 @@ describe('Governance Page', () => { 'proposal-details-requester-party-id' ); expect(requesterInput).toBeInTheDocument(); + // Resolve SV display name (e.g. Digital-Asset-2) to full party ID for display + copy. + expect( + within(votingInformationSection).getByTestId('proposal-details-requester-party-id-value') + .textContent + ).toMatch(/::/); const votingClosesIso = within(votingInformationSection).getByTestId( 'proposal-details-voting-closes-value' diff --git a/apps/sv/frontend/src/__tests__/governance/governance-sorting.test.tsx b/apps/sv/frontend/src/__tests__/governance/governance-sorting.test.tsx index fe1307b030..f593de50f4 100644 --- a/apps/sv/frontend/src/__tests__/governance/governance-sorting.test.tsx +++ b/apps/sv/frontend/src/__tests__/governance/governance-sorting.test.tsx @@ -62,7 +62,7 @@ describe('Governance Page Sorting', () => { }); }); - describe('Inflight Votes Section', () => { + describe('In-flight Proposals Section', () => { const baseData: Omit< ProposalListingData, 'actionName' | 'contractId' | 'voteTakesEffect' | 'votingThresholdDeadline' | 'voteStats' @@ -120,7 +120,7 @@ describe('Governance Page Sorting', () => { render( , @@ -160,16 +170,21 @@ describe('Proposal Details Content', () => { ); - const pageTitle = screen.getByTestId('proposal-details-title'); - expect(pageTitle.textContent).toMatch(/Proposal Details/); + expect(screen.getByTestId('proposal-details-title')).toHaveTextContent('Proposal Details'); + const backToAllVotes = screen.getByTestId('proposal-details-back-to-all-votes'); + expect(backToAllVotes).toHaveTextContent('Back to all votes'); + expect(backToAllVotes).toHaveAttribute('href', '/governance/proposals'); + + const proposalDetailsSection = screen.getByTestId('proposal-details-proposal-details'); + expect(proposalDetailsSection).toBeInTheDocument(); + // Figma starts at Action — no duplicate inner "Proposal Details" heading + expect( + within(proposalDetailsSection).queryByRole('heading', { name: 'Proposal Details' }) + ).toBeNull(); const action = screen.getByTestId('proposal-details-action-value'); expect(action.textContent).toMatch(/Offboard Member/); - expect(screen.getByTestId('proposal-details-contractid-label').textContent).toBe( - VOTE_PROPOSAL_CONTRACT_ID_LABEL - ); - const offboardSection = screen.getByTestId('proposal-details-offboard-member-section'); expect(offboardSection).toBeInTheDocument(); @@ -178,7 +193,16 @@ describe('Proposal Details Content', () => { ); expect(memberInput).toBeInTheDocument(); expect(memberInput.textContent).toBe('sv2'); + expect(within(offboardSection).getByTestId('proposal-details-member-party-id')).toHaveStyle({ + width: '100%', + }); + expect( + within(offboardSection).getByTestId('proposal-details-member-party-id-scroll') + ).toHaveStyle({ overflowX: 'auto', width: '100%' }); + expect(screen.getByTestId('proposal-details-summary-label').textContent).toBe( + PROPOSAL_SUMMARY_TITLE + ); const summary = screen.getByTestId('proposal-details-summary-value'); expect(summary.textContent).toMatch(/Summary of the proposal/); @@ -186,6 +210,32 @@ describe('Proposal Details Content', () => { const url = screen.getByTestId('proposal-details-url'); expect(url.textContent).toMatch(/https:\/\/example.com/); + expect(url).toHaveStyle({ width: '100%' }); + expect(screen.getByTestId('proposal-details-url-scroll')).toHaveStyle({ + overflowX: 'auto', + width: '100%', + }); + + // Figma Offboard details order: Action → Member → Proposal Summary → Supporting URL → Contract ID + expect(screen.getByTestId('proposal-details-contractid-label').textContent).toBe( + VOTE_PROPOSAL_CONTRACT_ID_LABEL + ); + expect(screen.getByTestId('proposal-details-contractid-id')).toHaveStyle({ width: '100%' }); + expect(screen.getByTestId('proposal-details-contractid-id-scroll')).toHaveStyle({ + overflowX: 'auto', + width: '100%', + }); + const contractIdLabel = screen.getByTestId('proposal-details-contractid-label'); + expect( + action.compareDocumentPosition(offboardSection) & Node.DOCUMENT_POSITION_FOLLOWING + ).toBeTruthy(); + expect( + offboardSection.compareDocumentPosition(summary) & Node.DOCUMENT_POSITION_FOLLOWING + ).toBeTruthy(); + expect(summary.compareDocumentPosition(url) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); + expect( + url.compareDocumentPosition(contractIdLabel) & Node.DOCUMENT_POSITION_FOLLOWING + ).toBeTruthy(); const votingInformationSection = screen.getByTestId('proposal-details-voting-information'); expect(votingInformationSection).toBeInTheDocument(); @@ -195,6 +245,26 @@ describe('Proposal Details Content', () => { ); expect(requesterInput).toBeInTheDocument(); expect(requesterInput.textContent).toBe('sv1'); + expect( + within(votingInformationSection).getByTestId('proposal-details-requester-party-id') + ).toHaveStyle({ width: '100%' }); + expect( + within(votingInformationSection).getByTestId('proposal-details-requester-party-id-scroll') + ).toHaveStyle({ overflowX: 'auto', width: '100%' }); + + expect(screen.getByTestId('proposal-details-created-at-label').textContent).toBe( + PROPOSAL_CREATED_LABEL + ); + expect(screen.getByTestId('proposal-details-created-at-value').textContent).toBe( + '2025-01-01 13:00' + ); + + expect(screen.getByTestId('proposal-details-threshold-deadline-label').textContent).toBe( + THRESHOLD_DEADLINE_LABEL + ); + expect(screen.getByTestId('proposal-details-effective-at-label').textContent).toBe( + EFFECTIVE_AT_LABEL + ); const votingClosesIso = within(votingInformationSection).getByTestId( 'proposal-details-voting-closes-value' @@ -217,7 +287,12 @@ describe('Proposal Details Content', () => { expect(screen.getByTestId('your-vote-form')).toBeInTheDocument(); expect(screen.getByTestId('your-vote-url-input')).toBeInTheDocument(); - expect(screen.getByTestId('your-vote-reason-input')).toBeInTheDocument(); + const reasonInput = screen.getByTestId('your-vote-reason-input'); + expect(reasonInput).toBeInTheDocument(); + expect(reasonInput.getAttribute('placeholder')).toBe(VOTE_REASON_PLACEHOLDER); + expect(screen.getByTestId('your-vote-url-input').getAttribute('placeholder')).toBe( + VOTE_REASON_URL_PLACEHOLDER + ); expect(screen.getByTestId('your-vote-accept')).toBeInTheDocument(); expect(screen.getByTestId('your-vote-reject')).toBeInTheDocument(); }); @@ -725,6 +800,42 @@ const votesData = [ ] as ProposalVote[]; describe('Proposal Details > Votes & Voting', () => { + test('should render vote URLs at the same column width as SV IDs', () => { + const voter = 'sv-party::1220abcdef1234567890abcdef'; + const url = 'https://example.com/a/long/vote/reason/url/that-exceeds-compact-width'; + + render( + + + + ); + + // Party IDs fill the vote-row text column (no 270px compact cap). + expect(screen.getByTestId('proposal-details-voter-party-id')).toHaveStyle({ width: '100%' }); + // Vote reason URLs match that column width; copy sits in the party-ID copy track. + expect(screen.getByTestId('proposal-details-vote-url')).toHaveStyle({ width: '100%' }); + expect(screen.getByTestId('proposal-details-vote-url-scroll')).toHaveStyle({ + overflowX: 'auto', + width: '100%', + }); + expect(screen.getByTestId('proposal-details-vote-url-copy-button')).toBeInTheDocument(); + const displayedUrl = screen.getByTestId('proposal-details-vote-url-link'); + expect(displayedUrl).toHaveAttribute('href', url); + expect(displayedUrl.textContent).toBe(url); + }); + test('should render votes table', () => { render( @@ -753,6 +864,14 @@ describe('Proposal Details > Votes & Voting', () => { expect(acceptedVotesTab.getAttribute('aria-selected')).toBe('false'); expect(rejectedVotesTab.getAttribute('aria-selected')).toBe('false'); expect(noVoteVotesTab.getAttribute('aria-selected')).toBe('false'); + + const voterScroll = screen.getAllByTestId('proposal-details-voter-party-id-scroll'); + expect(voterScroll.length).toBeGreaterThan(0); + expect(voterScroll[0]).toHaveStyle({ overflowX: 'auto' }); + // Votes rows fill width and scroll — no fixed 270px cap. + expect(screen.getAllByTestId('proposal-details-voter-party-id')[0]).toHaveStyle({ + width: '100%', + }); }); test('should filter votes by tabs', async () => { @@ -973,9 +1092,11 @@ describe('Proposal Details > Votes & Voting', () => { const votingFormUrlInput = within(votingForm).getByTestId('your-vote-url-input'); expect(votingFormUrlInput).toBeInTheDocument(); + expect(votingFormUrlInput).toHaveAttribute('placeholder', URL_PLACEHOLDER); const votingFormReasonInput = within(votingForm).getByTestId('your-vote-reason-input'); expect(votingFormReasonInput).toBeInTheDocument(); + expect(votingFormReasonInput).toHaveAttribute('placeholder', VOTE_REASON_PLACEHOLDER); const votingFormAccept = within(votingForm).getByTestId('your-vote-accept'); expect(votingFormAccept).toBeInTheDocument(); @@ -1026,6 +1147,12 @@ describe('Proposal Details > Votes & Voting', () => { expect(acceptButton.textContent).toMatch(/Accept/); expect(rejectButton).toBeInTheDocument(); expect(rejectButton.textContent).toMatch(/Reject/); + // Figma / #6912: Reject (left) → Accept (right); primary on the right + const voteButtons = within(votingForm).getAllByRole('button'); + expect(voteButtons.map(b => b.getAttribute('data-testid'))).toEqual([ + 'your-vote-reject', + 'your-vote-accept', + ]); }); test('render success message after api returns success', async () => { diff --git a/apps/sv/frontend/src/__tests__/governance/proposal-summary.test.tsx b/apps/sv/frontend/src/__tests__/governance/proposal-summary.test.tsx index 74beb26805..7eca1496df 100644 --- a/apps/sv/frontend/src/__tests__/governance/proposal-summary.test.tsx +++ b/apps/sv/frontend/src/__tests__/governance/proposal-summary.test.tsx @@ -3,6 +3,13 @@ import { render, screen } from '@testing-library/react'; import { describe, expect, test } from 'vitest'; import { ProposalSummary } from '../../components/governance/ProposalSummary'; +import { + CREATE_PROPOSAL_LABEL_PROPOSAL_TYPE, + EFFECTIVE_AT_LABEL, + PROPOSAL_REVIEW_TITLE, + SUPPORTING_URL_LABEL, + THRESHOLD_DEADLINE_LABEL, +} from '../../utils/constants'; import { ConfigChange } from '../../utils/types'; const url = 'https://example.com'; @@ -10,6 +17,29 @@ const summary = 'Summary of the proposal'; const expiryDate = '2025-09-25 11:00'; const effectiveDate = '2025-09-26 11:00'; +/** Shared labels for the post-rebase ProposalSummary / ProposalReviewField chrome. */ +const REVIEW_LABELS = { + title: PROPOSAL_REVIEW_TITLE, + action: CREATE_PROPOSAL_LABEL_PROPOSAL_TYPE, + expiryDate: THRESHOLD_DEADLINE_LABEL, + effectiveDate: EFFECTIVE_AT_LABEL, + summary: 'Proposal Summary', + url: SUPPORTING_URL_LABEL, +} as const; + +function expectCommonReviewFields(actionName: string) { + expect(screen.getByTestId('proposal-review-title').textContent).toBe(REVIEW_LABELS.title); + expect(screen.getByTestId('action-title').textContent).toBe(REVIEW_LABELS.action); + expect(screen.getByTestId('action-field').textContent).toBe(actionName); + expect(screen.getByTestId('url-title').textContent).toBe(REVIEW_LABELS.url); + expect(screen.getByTestId('url-field').textContent).toBe(url); + expect(screen.getByTestId('summary-title').textContent).toBe(REVIEW_LABELS.summary); + expect(screen.getByTestId('summary-field').textContent).toBe(summary); + expect(screen.getByTestId('expiryDate-title').textContent).toBe(REVIEW_LABELS.expiryDate); + expect(screen.getByTestId('expiryDate-field').textContent).toBe(expiryDate); + expect(screen.getByTestId('effectiveDate-title').textContent).toBe(REVIEW_LABELS.effectiveDate); +} + describe('Review Proposal Component', () => { test('should render review proposal component for offboard member', () => { const actionName = 'Offboard Member'; @@ -29,23 +59,12 @@ describe('Review Proposal Component', () => { /> ); - expect(screen.getByTestId('action-title').textContent).toBe('Action'); - expect(screen.getByTestId('action-field').textContent).toBe(actionName); - - expect(screen.getByTestId('url-title').textContent).toBe('Supporting URL'); - expect(screen.getByTestId('url-field').textContent).toBe(url); - - expect(screen.getByTestId('summary-title').textContent).toBe('Summary'); - expect(screen.getByTestId('summary-field').textContent).toBe(summary); - - expect(screen.getByTestId('expiryDate-title').textContent).toBe('Quorum Threshold Deadline'); - expect(screen.getByTestId('expiryDate-field').textContent).toBe(expiryDate); - - expect(screen.getByTestId('effectiveDate-title').textContent).toBe('Effective Date'); + expectCommonReviewFields(actionName); expect(screen.getByTestId('effectiveDate-field').textContent).toBe(effectiveDate); - expect(screen.getByTestId('offboardMember-title').textContent).toBe('Offboard Member'); - expect(screen.getByTestId('offboardMember-field').textContent).toBe(offboardMember); + expect(screen.getByTestId('offboardMember-title').textContent).toBe('Member'); + expect(screen.getByTestId('offboardMember-party-id-value').textContent).toBe(offboardMember); + expect(screen.getByTestId('offboardMember-party-id-copy-button')).toBeInTheDocument(); }); test('should render review proposal component for offboard member at Threshold', () => { @@ -66,7 +85,7 @@ describe('Review Proposal Component', () => { /> ); - expect(screen.getByTestId('effectiveDate-title').textContent).toBe('Effective Date'); + expect(screen.getByTestId('effectiveDate-title').textContent).toBe(EFFECTIVE_AT_LABEL); expect(screen.getByTestId('effectiveDate-field').textContent).toBe('Threshold'); }); @@ -93,21 +112,16 @@ describe('Review Proposal Component', () => { /> ); - expect(screen.getByTestId('action-title').textContent).toBe('Action'); - expect(screen.getByTestId('action-field').textContent).toBe(actionName); - - expect(screen.getByTestId('url-title').textContent).toBe('Supporting URL'); - expect(screen.getByTestId('url-field').textContent).toBe(url); - - expect(screen.getByTestId('summary-title').textContent).toBe('Summary'); - expect(screen.getByTestId('summary-field').textContent).toBe(summary); - - expect(screen.getByTestId('expiryDate-title').textContent).toBe('Quorum Threshold Deadline'); - expect(screen.getByTestId('expiryDate-field').textContent).toBe(expiryDate); - - expect(screen.getByTestId('effectiveDate-title').textContent).toBe('Effective Date'); + expectCommonReviewFields(actionName); expect(screen.getByTestId('effectiveDate-field').textContent).toBe(effectiveDate); + expect(screen.getByTestId('svRewardWeightMember-title').textContent).toBe('Member'); + expect(screen.getByTestId('svRewardWeightMember-party-id-value').textContent).toBe( + svRewardWeightMember + ); + expect(screen.getByTestId('svRewardWeightMember-party-id-copy-button')).toBeInTheDocument(); + + expect(screen.getByTestId('configChange-title').textContent).toBe('Proposed Changes'); expect(screen.getByTestId('config-change-field-label').textContent).toBe(title); expect(screen.getByTestId('config-change-current-value').textContent).toBe(currentWeight); expect(screen.getByTestId('config-change-new-value').textContent).toBe(svRewardWeight); @@ -133,23 +147,12 @@ describe('Review Proposal Component', () => { /> ); - expect(screen.getByTestId('action-title').textContent).toBe('Action'); - expect(screen.getByTestId('action-field').textContent).toBe(actionName); - - expect(screen.getByTestId('url-title').textContent).toBe('Supporting URL'); - expect(screen.getByTestId('url-field').textContent).toBe(url); - - expect(screen.getByTestId('summary-title').textContent).toBe('Summary'); - expect(screen.getByTestId('summary-field').textContent).toBe(summary); - - expect(screen.getByTestId('expiryDate-title').textContent).toBe('Quorum Threshold Deadline'); - expect(screen.getByTestId('expiryDate-field').textContent).toBe(expiryDate); - - expect(screen.getByTestId('effectiveDate-title').textContent).toBe('Effective Date'); + expectCommonReviewFields(actionName); expect(screen.getByTestId('effectiveDate-field').textContent).toBe(effectiveDate); expect(screen.getByTestId('grantRight-title').textContent).toBe('Provider Party ID'); - expect(screen.getByTestId('grantRight-field').textContent).toBe(provider); + expect(screen.getByTestId('grantRight-party-id-value').textContent).toBe(provider); + expect(screen.getByTestId('grantRight-party-id-copy-button')).toBeInTheDocument(); expect(screen.getByTestId('grantRightActivityWeight-title').textContent).toBe( 'Activity Weight' @@ -177,23 +180,14 @@ describe('Review Proposal Component', () => { /> ); - expect(screen.getByTestId('action-title').textContent).toBe('Action'); - expect(screen.getByTestId('action-field').textContent).toBe(actionName); - - expect(screen.getByTestId('url-title').textContent).toBe('Supporting URL'); - expect(screen.getByTestId('url-field').textContent).toBe(url); - - expect(screen.getByTestId('summary-title').textContent).toBe('Summary'); - expect(screen.getByTestId('summary-field').textContent).toBe(summary); - - expect(screen.getByTestId('expiryDate-title').textContent).toBe('Quorum Threshold Deadline'); - expect(screen.getByTestId('expiryDate-field').textContent).toBe(expiryDate); - - expect(screen.getByTestId('effectiveDate-title').textContent).toBe('Effective Date'); + expectCommonReviewFields(actionName); expect(screen.getByTestId('effectiveDate-field').textContent).toBe(effectiveDate); expect(screen.getByTestId('revokeProviderPartyId-title').textContent).toBe('Provider Party ID'); - expect(screen.getByTestId('revokeProviderPartyId-field').textContent).toBe(providerPartyId); + expect(screen.getByTestId('revokeProviderPartyId-party-id-value').textContent).toBe( + providerPartyId + ); + expect(screen.getByTestId('revokeProviderPartyId-party-id-copy-button')).toBeInTheDocument(); expect(screen.getByTestId('revokeRight-title').textContent).toBe( 'Featured Application Contract ID' @@ -225,33 +219,27 @@ describe('Review Proposal Component', () => { /> ); - expect(screen.getByTestId('action-title').textContent).toBe('Action'); - expect(screen.getByTestId('action-field').textContent).toBe(actionName); - - expect(screen.getByTestId('url-title').textContent).toBe('Supporting URL'); - expect(screen.getByTestId('url-field').textContent).toBe(url); - - expect(screen.getByTestId('summary-title').textContent).toBe('Summary'); - expect(screen.getByTestId('summary-field').textContent).toBe(summary); - - expect(screen.getByTestId('expiryDate-title').textContent).toBe('Quorum Threshold Deadline'); - expect(screen.getByTestId('expiryDate-field').textContent).toBe(expiryDate); - - expect(screen.getByTestId('effectiveDate-title').textContent).toBe('Effective Date'); + expectCommonReviewFields(actionName); expect(screen.getByTestId('effectiveDate-field').textContent).toBe(effectiveDate); expect(screen.getByTestId('updateProviderPartyId-title').textContent).toBe('Provider Party ID'); - expect(screen.getByTestId('updateProviderPartyId-field').textContent).toBe(providerPartyId); + expect(screen.getByTestId('updateProviderPartyId-party-id-value').textContent).toBe( + providerPartyId + ); + expect(screen.getByTestId('updateProviderPartyId-party-id-copy-button')).toBeInTheDocument(); expect(screen.getByTestId('updateRight-title').textContent).toBe( 'Featured Application Contract ID' ); expect(screen.getByTestId('updateRight-field').textContent).toBe(rightCid); + expect(screen.getByTestId('updateActivityWeight-title').textContent).toBe('Proposed Changes'); expect(screen.getByTestId('config-change-current-value').textContent).toBe( currentActivityWeight ); expect(screen.getByTestId('config-change-new-value').textContent).toBe(newActivityWeight); + + expect(screen.queryByTestId('updateReason-field')).not.toBeInTheDocument(); }); test('should render review proposal component for dso rules config', () => { @@ -288,22 +276,12 @@ describe('Review Proposal Component', () => { /> ); - expect(screen.getByTestId('action-title').textContent).toBe('Action'); - expect(screen.getByTestId('action-field').textContent).toBe(actionName); - - expect(screen.getByTestId('url-title').textContent).toBe('Supporting URL'); - expect(screen.getByTestId('url-field').textContent).toBe(url); - - expect(screen.getByTestId('summary-title').textContent).toBe('Summary'); - expect(screen.getByTestId('summary-field').textContent).toBe(summary); - - expect(screen.getByTestId('expiryDate-title').textContent).toBe('Quorum Threshold Deadline'); - expect(screen.getByTestId('expiryDate-field').textContent).toBe(expiryDate); - - expect(screen.getByTestId('effectiveDate-title').textContent).toBe('Effective Date'); + expectCommonReviewFields(actionName); expect(screen.getByTestId('effectiveDate-field').textContent).toBe(effectiveDate); - expect(screen.getByText('Proposed Changes')).toBeDefined(); + expect(screen.getByTestId('configChange-title').textContent).toBe( + 'Proposed Configuration Changes' + ); expect(screen.getByText(numThresholdTitle)).toBeDefined(); expect(screen.getByText(voteCooldownTitle)).toBeDefined(); @@ -360,22 +338,12 @@ describe('Review Proposal Component', () => { /> ); - expect(screen.getByTestId('action-title').textContent).toBe('Action'); - expect(screen.getByTestId('action-field').textContent).toBe(actionName); - - expect(screen.getByTestId('url-title').textContent).toBe('Supporting URL'); - expect(screen.getByTestId('url-field').textContent).toBe(url); - - expect(screen.getByTestId('summary-title').textContent).toBe('Summary'); - expect(screen.getByTestId('summary-field').textContent).toBe(summary); - - expect(screen.getByTestId('expiryDate-title').textContent).toBe('Quorum Threshold Deadline'); - expect(screen.getByTestId('expiryDate-field').textContent).toBe(expiryDate); - - expect(screen.getByTestId('effectiveDate-title').textContent).toBe('Effective Date'); + expectCommonReviewFields(actionName); expect(screen.getByTestId('effectiveDate-field').textContent).toBe(effectiveDate); - expect(screen.getByText('Proposed Changes')).toBeDefined(); + expect(screen.getByTestId('configChange-title').textContent).toBe( + 'Proposed Configuration Changes' + ); expect(screen.getByText(feeTitle)).toBeDefined(); expect(screen.getByText(feeRateTitle)).toBeDefined(); diff --git a/apps/sv/frontend/src/__tests__/layout/sv-top-nav.test.tsx b/apps/sv/frontend/src/__tests__/layout/sv-top-nav.test.tsx new file mode 100644 index 0000000000..4f0d14f221 --- /dev/null +++ b/apps/sv/frontend/src/__tests__/layout/sv-top-nav.test.tsx @@ -0,0 +1,35 @@ +// Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +import { render, screen } from '@testing-library/react'; +import { MemoryRouter } from 'react-router'; +import { describe, expect, test, vi } from 'vitest'; + +import SvTopNav from '../../components/layout/SvTopNav'; + +const navLinks = [ + { name: 'Global Synchronizer Information', path: '/dso' }, + { name: 'Governance', path: '/governance' }, + { name: 'Teluma Price', path: '/amulet-price' }, + { name: 'Validators', path: '/validator-onboarding' }, +]; + +describe('SvTopNav', () => { + test('renders brand, centered nav cluster, and logout', () => { + render( + + + + ); + + expect(screen.getByTestId('app-title')).toHaveTextContent('Supervalidator Operations'); + expect(screen.getByTestId('sv-top-nav-links')).toBeInTheDocument(); + expect(screen.getByTestId('navlink-dso')).toBeInTheDocument(); + expect(screen.getByTestId('navlink-governance')).toBeInTheDocument(); + expect(screen.getByTestId('logout-button')).toBeInTheDocument(); + + const row = screen.getByTestId('sv-top-nav'); + expect(row).toHaveStyle({ display: 'flex' }); + expect(screen.getByTestId('sv-top-nav-spacer-start')).toBeInTheDocument(); + expect(screen.getByTestId('sv-top-nav-spacer-end')).toBeInTheDocument(); + }); +}); diff --git a/apps/sv/frontend/src/__tests__/mocks/handlers/sv-api.ts b/apps/sv/frontend/src/__tests__/mocks/handlers/sv-api.ts index e335af9bf9..e707d63089 100644 --- a/apps/sv/frontend/src/__tests__/mocks/handlers/sv-api.ts +++ b/apps/sv/frontend/src/__tests__/mocks/handlers/sv-api.ts @@ -36,7 +36,7 @@ export const buildSvMock = (svUrl: string): HttpHandler[] => [ return new HttpResponse(null, { status: 200 }); }), - dsoInfoHandler(svUrl), + dsoInfoHandler(svUrl, '/v1/dso'), http.get(`${svUrl}/v0/admin/sv/voterequests`, () => { return HttpResponse.json(voteRequests); diff --git a/apps/sv/frontend/src/__tests__/synchroniser-upgrade.test.tsx b/apps/sv/frontend/src/__tests__/synchroniser-upgrade.test.tsx index 28909ab174..0cf8207b68 100644 --- a/apps/sv/frontend/src/__tests__/synchroniser-upgrade.test.tsx +++ b/apps/sv/frontend/src/__tests__/synchroniser-upgrade.test.tsx @@ -66,7 +66,7 @@ describe('SV user can', () => { test('set next scheduled synchronizer upgrade', async () => { server.use( - http.get(`${svUrl}/v0/dso`, () => { + http.get(`${svUrl}/v1/dso`, () => { return HttpResponse.json(dsoInfoWithoutSynchronizerUpgrade); }) ); @@ -94,7 +94,7 @@ describe('SV user can', () => { test('submit vote request with new valid synchronizer upgrade time', async () => { server.use( - http.get(`${svUrl}/v0/dso`, () => { + http.get(`${svUrl}/v1/dso`, () => { return HttpResponse.json(dsoInfoWithoutSynchronizerUpgrade); }) ); @@ -142,7 +142,7 @@ describe('SV user can', () => { test('submit vote request with existing and unchanged synchronizer upgrade time', async () => { server.use( - http.get(`${svUrl}/v0/dso`, () => { + http.get(`${svUrl}/v1/dso`, () => { return HttpResponse.json(dsoInfoWithSynchronizerUpgrade); }) ); @@ -169,7 +169,7 @@ describe('SV user can', () => { test('not submit vote request if new synchronizer upgrade time is before expiry', async () => { server.use( - http.get(`${svUrl}/v0/dso`, () => { + http.get(`${svUrl}/v1/dso`, () => { return HttpResponse.json(dsoInfoWithoutSynchronizerUpgrade); }) ); @@ -225,7 +225,7 @@ describe('SV user can', () => { test('not submit vote request if synchronizer upgrade time is changed and is before expiry and effective at threshold', async () => { server.use( - http.get(`${svUrl}/v0/dso`, () => { + http.get(`${svUrl}/v1/dso`, () => { return HttpResponse.json(dsoInfoWithSynchronizerUpgrade); }) ); @@ -282,7 +282,7 @@ describe('SV user can', () => { test('not submit vote request if synchronizer upgrade time is changed and is before effective date', async () => { server.use( - http.get(`${svUrl}/v0/dso`, () => { + http.get(`${svUrl}/v1/dso`, () => { return HttpResponse.json(dsoInfoWithSynchronizerUpgrade); }) ); @@ -339,7 +339,7 @@ describe('SV user can', () => { 'make changes with different timezones', async () => { server.use( - http.get(`${svUrl}/v0/dso`, () => { + http.get(`${svUrl}/v1/dso`, () => { return HttpResponse.json(dsoInfoWithoutSynchronizerUpgrade); }) ); diff --git a/apps/sv/frontend/src/components/Layout.tsx b/apps/sv/frontend/src/components/Layout.tsx index bbfca185e7..c119706a6f 100644 --- a/apps/sv/frontend/src/components/Layout.tsx +++ b/apps/sv/frontend/src/components/Layout.tsx @@ -59,7 +59,7 @@ const Layout: React.FC = ({ children }) => { } const navLinks: SvNavLinkItem[] = [ - { name: 'Global Synchronizer Information', path: '/dso' }, + { name: 'Global Synchronizer Information', path: '/dso', alsoActiveFor: ['/'] }, { name: 'Governance', path: '/governance', diff --git a/apps/sv/frontend/src/components/beta/CopyableIdentifier.tsx b/apps/sv/frontend/src/components/beta/CopyableIdentifier.tsx index 1896cf0d47..a8b60c012a 100644 --- a/apps/sv/frontend/src/components/beta/CopyableIdentifier.tsx +++ b/apps/sv/frontend/src/components/beta/CopyableIdentifier.tsx @@ -5,76 +5,153 @@ import { Box, Chip, IconButton, Typography } from '@mui/material'; import { useRef } from 'react'; import { useHorizontalScrollMetrics } from '../../hooks/useHorizontalScrollMetrics'; -import { scrollContainerSx, scrollTextSx, scrollThumbSx, scrollTrackSx } from './identifierStyles'; +import { + ellipsisContainerSx, + ellipsisTextSx, + IDENTIFIER_COMPACT_MAX_WIDTH_PX, + scrollContainerSx, + scrollTextSx, + scrollThumbSx, + scrollTrackSx, +} from './identifierStyles'; export type CopyableIdentifierSize = 'small' | 'large'; +export type CopyableIdentifierOverflow = 'scroll' | 'ellipsis'; interface CopyableIdentifierProps { value: string; copyValue?: string; badge?: string; size: CopyableIdentifierSize; + overflow?: CopyableIdentifierOverflow; + /** + * Caps the text slot (Figma ~270px). With `overflow="scroll"`, the ID stays + * horizontally scrollable inside the cap (#1785 + Figma width). With + * `overflow="ellipsis"`, CSS ellipsis is used instead. + */ + maxWidth?: number; + /** + * Fill the parent width: party-ID text flexes/scrolls; copy + badge stay fixed. + * Used by Votes rows (ID + reason shrink; status stays right-aligned). + */ + fullWidth?: boolean; + /** When true, only the (scrollable) value is rendered — caller places copy / badge. */ + hideCopy?: boolean; 'data-testid': string; } +/** Gap between party-ID text and copy / You accessories. */ +const IDENTIFIER_ACCESSORY_GAP = '8px'; + const CopyableIdentifier: React.FC = ({ value, copyValue, badge, size, + overflow = 'scroll', + maxWidth, + fullWidth = false, + hideCopy = false, 'data-testid': testId, }) => { const scrollRef = useRef(null); - const metrics = useHorizontalScrollMetrics(scrollRef, [value]); - const fontSize = size === 'small' ? '14px' : '18px'; + const metrics = useHorizontalScrollMetrics(scrollRef, [value, maxWidth, fullWidth]); + const fontSize = size === 'small' ? '14px' : '16px'; + const isEllipsis = overflow === 'ellipsis'; + const compactMaxWidth = maxWidth ?? (isEllipsis ? IDENTIFIER_COMPACT_MAX_WIDTH_PX : undefined); + const showAccessories = !hideCopy; return ( - - + + {value} - {metrics.canScroll && ( + {!isEllipsis && metrics.canScroll && ( )} - { - e.stopPropagation(); - e.preventDefault(); - navigator.clipboard.writeText(copyValue ?? value); - }} - > - - - {badge !== undefined && ( - + {showAccessories && ( + <> + { + e.stopPropagation(); + e.preventDefault(); + navigator.clipboard.writeText(copyValue ?? value); + }} + > + + + {badge !== undefined && ( + + )} + )} ); diff --git a/apps/sv/frontend/src/components/beta/CopyableUrl.tsx b/apps/sv/frontend/src/components/beta/CopyableUrl.tsx index 827bfa9a7f..bb0fc28d09 100644 --- a/apps/sv/frontend/src/components/beta/CopyableUrl.tsx +++ b/apps/sv/frontend/src/components/beta/CopyableUrl.tsx @@ -4,51 +4,115 @@ import { ContentCopy } from '@mui/icons-material'; import { Box, IconButton, Link } from '@mui/material'; import { sanitizeUrl } from '@canton-network/splice-common-frontend-utils'; +import { useRef } from 'react'; +import { useHorizontalScrollMetrics } from '../../hooks/useHorizontalScrollMetrics'; import type { CopyableIdentifierSize } from './CopyableIdentifier'; +import { + scrollContainerSx, + scrollThumbSx, + scrollTrackSx, + URL_COMPACT_MAX_WIDTH_PX, +} from './identifierStyles'; interface CopyableUrlProps { url: string; size: CopyableIdentifierSize; + /** + * Fill the parent width (proposal-details section / Votes row). Default keeps + * the compact Supporting URL slot (~346px). + */ + fullWidth?: boolean; + /** When true, only the (scrollable) link is rendered — caller places copy. */ + hideCopy?: boolean; 'data-testid': string; } -function abbreviateUrl(url: string, maxLength = 50): string { - if (url.length <= maxLength) { - return url; - } - return `${url.slice(0, maxLength)}...`; -} - -const CopyableUrl: React.FC = ({ url, size, 'data-testid': testId }) => { +const CopyableUrl: React.FC = ({ + url, + size, + fullWidth = false, + hideCopy = false, + 'data-testid': testId, +}) => { const sanitizedUrl = sanitizeUrl(url); + const fontSize = size === 'small' ? '14px' : '16px'; + const scrollRef = useRef(null); + const metrics = useHorizontalScrollMetrics(scrollRef, [sanitizedUrl, fullWidth, hideCopy]); + const textMaxWidth = fullWidth ? '100%' : URL_COMPACT_MAX_WIDTH_PX; + const showCopy = !hideCopy; return ( - - + - {abbreviateUrl(sanitizedUrl)} - - navigator.clipboard.writeText(sanitizedUrl)} > - - + + + {sanitizedUrl} + + + {metrics.canScroll && ( + + + + )} + + {showCopy && ( + navigator.clipboard.writeText(sanitizedUrl)} + > + + + )} ); }; diff --git a/apps/sv/frontend/src/components/beta/MemberIdentifier.tsx b/apps/sv/frontend/src/components/beta/MemberIdentifier.tsx index 80a50ac7ac..b51cbe8cba 100644 --- a/apps/sv/frontend/src/components/beta/MemberIdentifier.tsx +++ b/apps/sv/frontend/src/components/beta/MemberIdentifier.tsx @@ -2,12 +2,15 @@ // SPDX-License-Identifier: Apache-2.0 import CopyableIdentifier from './CopyableIdentifier'; -import type { CopyableIdentifierSize } from './CopyableIdentifier'; +import type { CopyableIdentifierOverflow, CopyableIdentifierSize } from './CopyableIdentifier'; interface MemberIdentifierProps { partyId: string; isYou: boolean; size: CopyableIdentifierSize; + overflow?: CopyableIdentifierOverflow; + maxWidth?: number; + fullWidth?: boolean; 'data-testid': string; } @@ -15,6 +18,9 @@ const MemberIdentifier: React.FC = ({ partyId, isYou, size, + overflow, + maxWidth, + fullWidth, 'data-testid': testId, }) => ( = ({ copyValue={partyId} badge={isYou ? 'You' : undefined} size={size} + overflow={overflow} + maxWidth={maxWidth} + fullWidth={fullWidth} data-testid={testId} /> ); diff --git a/apps/sv/frontend/src/components/beta/identifierStyles.ts b/apps/sv/frontend/src/components/beta/identifierStyles.ts index 6b3ea34c67..ec59d01c9d 100644 --- a/apps/sv/frontend/src/components/beta/identifierStyles.ts +++ b/apps/sv/frontend/src/components/beta/identifierStyles.ts @@ -20,15 +20,42 @@ export const scrollContainerSx: SxProps = { export const scrollTextSx: SxProps = { display: 'inline-block', - width: 'max-content', - minWidth: '100%', whiteSpace: 'nowrap', textOverflow: 'clip', + // Intrinsic width for overflow scroll; parent must clip (minmax(0,1fr) / overflow). + width: 'max-content', +}; + +export const ellipsisContainerSx: SxProps = { + minWidth: 0, + // Figma truncated ID text slot (e.g. Vote proposal contract id Group 461): 270px + maxWidth: 270, + width: '100%', + overflow: 'hidden', +}; + +/** Figma Group 461 truncated ID text width — also used to cap scrollable compact IDs. */ +export const IDENTIFIER_COMPACT_MAX_WIDTH_PX = 270; + +/** Figma Supporting URL value slot (`552:960`). */ +export const URL_COMPACT_MAX_WIDTH_PX = 346; + +export const ellipsisTextSx: SxProps = { + display: 'block', + overflow: 'hidden', + textOverflow: 'ellipsis', + whiteSpace: 'nowrap', + maxWidth: '100%', }; export const scrollableIdentifierFieldSx: SxProps = { fontFamily: 'Source Code Pro, monospace', - ...scrollTextSx, + display: 'inline-block', + width: 'max-content', + minWidth: '100%', + maxWidth: '100%', + whiteSpace: 'nowrap', + textOverflow: 'clip', }; const scrollableInputTextSx = { diff --git a/apps/sv/frontend/src/components/form-components/ConfigField.tsx b/apps/sv/frontend/src/components/form-components/ConfigField.tsx index de91c9786b..ea6dc53a57 100644 --- a/apps/sv/frontend/src/components/form-components/ConfigField.tsx +++ b/apps/sv/frontend/src/components/form-components/ConfigField.tsx @@ -17,6 +17,10 @@ import { useFieldContext } from '../../hooks/formContext'; import type { ConfigChange, PendingConfigFieldInfo } from '../../utils/types'; import { nextScheduledSynchronizerUpgradeFormat } from '@canton-network/splice-common-frontend-utils'; import { configFieldFieldSx, configFieldInputSx } from '../../themes/fieldStyles'; +import { + CREATE_PROPOSAL_CONFIG_INPUT_WIDTH, + CREATE_PROPOSAL_FIELD_BODY_SX, +} from '../../constants/createProposalLayout'; dayjs.extend(relativeTime); @@ -83,27 +87,54 @@ export const ConfigField: React.FC = props => { <> - - + + {configChange.label} {configChange.fieldName} - + {configChange.options ? (