Merge branch 'feature-cip-73' into dn/cip-0073-daml-weighted-validator-license

[ci]

Signed-off-by: Simon Meier <[email protected]>

Author: meiersi-da

.github/workflows/build.yml

@@ -165,7 +165,7 @@ jobs:
       test_names_file: 'test-full-class-names.log'
       start_canton_options: -w
       # TODO(DACH-NY/canton-network-node#10912) Investigate why things got slower
-      parallelism: 10
+      parallelism: 11
       test_name: wall-clock-time
       with_gcp_creds: true
       skip_if_regex: "\\[bft\\]"

LATEST_RELEASE

@@ -1 +1 @@
-0.4.22
+0.4.23

apps/app/src/main/scala/org/lfdecentralizedtrust/splice/config/SpliceConfig.scala

@@ -420,6 +420,7 @@ object SpliceConfig {
 
     implicit val upgradesConfig: ConfigReader[UpgradesConfig] = deriveReader[UpgradesConfig]
 
+    implicit val ingestionConfig: ConfigReader[IngestionConfig] = deriveReader[IngestionConfig]
     implicit val automationConfig: ConfigReader[AutomationConfig] =
       deriveReader[AutomationConfig]
     implicit val LedgerApiClientConfigReader: ConfigReader[LedgerApiClientConfig] =
@@ -688,13 +689,20 @@ object SpliceConfig {
       }
     implicit val migrateValidatorPartyConfigReader: ConfigReader[MigrateValidatorPartyConfig] =
       deriveReader[MigrateValidatorPartyConfig]
-    implicit val participantPruningConfigReader: ConfigReader[ParticipantPruningConfig] =
-      deriveReader[ParticipantPruningConfig]
+    implicit val pruningConfigReader: ConfigReader[PruningConfig] =
+      deriveReader[PruningConfig]
     implicit val validatorConfigReader: ConfigReader[ValidatorAppBackendConfig] =
       deriveReader[ValidatorAppBackendConfig].emap { conf =>
         val participantIdentifier =
           ValidatorCantonIdentifierConfig.resolvedNodeIdentifierConfig(conf).participant
         for {
+          _ <- Either.cond(
+            !(conf.domains.global.url.isDefined && conf.domains.global.sequencerNames.isDefined),
+            (),
+            ConfigValidationFailed(
+              "Configuration error: `url` and `sequencerNames` cannot both be specified for the global domain."
+            ),
+          )
           _ <- Either.cond(
             !conf.svValidator || conf.validatorPartyHint.isEmpty,
             (),
@@ -846,6 +854,7 @@ object SpliceConfig {
 
     implicit val upgradesConfig: ConfigWriter[UpgradesConfig] = deriveWriter[UpgradesConfig]
 
+    implicit val ingestionConfig: ConfigWriter[IngestionConfig] = deriveWriter[IngestionConfig]
     implicit val automationConfig: ConfigWriter[AutomationConfig] =
       deriveWriter[AutomationConfig]
     implicit val LedgerApiClientConfigWriter: ConfigWriter[LedgerApiClientConfig] =
@@ -1022,8 +1031,8 @@ object SpliceConfig {
       deriveWriter[TransferPreapprovalConfig]
     implicit val migrateValidatorPartyConfigWriter: ConfigWriter[MigrateValidatorPartyConfig] =
       deriveWriter[MigrateValidatorPartyConfig]
-    implicit val participantPruningConfigWriter: ConfigWriter[ParticipantPruningConfig] =
-      deriveWriter[ParticipantPruningConfig]
+    implicit val pruningConfigWriter: ConfigWriter[PruningConfig] =
+      deriveWriter[PruningConfig]
     implicit val validatorConfigWriter: ConfigWriter[ValidatorAppBackendConfig] =
       deriveWriter[ValidatorAppBackendConfig]
     implicit val validatorClientConfigWriter: ConfigWriter[ValidatorAppClientConfig] =

apps/app/src/main/scala/org/lfdecentralizedtrust/splice/console/SpliceInstanceReference.scala

@@ -258,20 +258,20 @@ class ParticipantClientReference(
     val pkgs = this.ledger_api.packages.list()
     if (!pkgs.map(_.packageId).contains(hash)) {
       discard[String](this.dars.upload(path, vetAllPackages = false))
-      val connected = this.synchronizers.list_connected()
-      if (connected.isEmpty) {
-        logger.error(s"Trying to vet $path on ${this.id} but not connected to any synchronizer")
-      }
-      connected.foreach { sync =>
-        this.topology.vetted_packages.propose_delta(
-          this.id,
-          adds = dar.all
-            .map(p => LfPackageId.assertFromString(p.getHash))
-            .distinct
-            .map(VettedPackage(_, None, None)),
-          store = TopologyStoreId.Synchronizer(sync.synchronizerId),
-        )
-      }
+    }
+    val connected = this.synchronizers.list_connected()
+    if (connected.isEmpty) {
+      logger.error(s"Trying to vet $path on ${this.id} but not connected to any synchronizer")
+    }
+    connected.foreach { sync =>
+      this.topology.vetted_packages.propose_delta(
+        this.id,
+        adds = dar.all
+          .map(p => LfPackageId.assertFromString(p.getHash))
+          .distinct
+          .map(VettedPackage(_, None, None)),
+        store = TopologyStoreId.Synchronizer(sync.synchronizerId),
+      )
     }
   }
 }

apps/app/src/test/resources/include/participants.conf

@@ -76,9 +76,11 @@ _participant_template {
     }
     # Default expiration used in tests
     max-token-lifetime = 30 days
+    # Required for pruning
+    admin-token-config.admin-claim=true
   }
 
-  admin-api.limits.pending {
+  admin-api.limits.active {
     "com.digitalasset.canton.admin.participant.v30.ParticipantRepairService/ExportAcsOld": 1,
     "com.digitalasset.canton.admin.participant.v30.PartyManagementService/ExportAcsOld": 1,
     "com.digitalasset.canton.admin.participant.v30.ParticipantRepairService/ImportAcsOld": 1,

apps/app/src/test/resources/include/sequencers.conf

@@ -48,20 +48,20 @@ _sequencer_reference_template {
   # so that sequencer will not block other sequencers from pruning even there is lack of activities
   # ref: https://github.com/DACH-NY/canton/issues/16371#issuecomment-1885005687
   time-tracker.min-observation-duration = 10s
-  public-api.limits.pending = {
+  public-api.limits.active = {
     "com.digitalasset.canton.sequencer.api.v30.SequencerService/DownloadTopologyStateForInit" : 3,
     "com.digitalasset.canton.sequencer.api.v30.SequencerService/DownloadTopologyStateForInitHash" : 3,
     "com.digitalasset.canton.sequencer.api.v30.SequencerService/Subscribe" : 1000,
   }
-  admin-api.limits.pending {
+  admin-api.limits.active {
     "com.digitalasset.canton.topology.admin.v30.TopologyManagerReadService/GenesisState": 1,
     "com.digitalasset.canton.topology.admin.v30.TopologyManagerReadService/GenesisStateV2": 1,
     "com.digitalasset.canton.topology.admin.v30.TopologyManagerReadService/ExportTopologySnapshot": 1,
     "com.digitalasset.canton.topology.admin.v30.TopologyManagerReadService/ExportTopologySnapshotV2": 1,
     "com.digitalasset.canton.sequencer.admin.v30.SequencerAdministrationService/OnboardingState": 1,
     "com.digitalasset.canton.sequencer.admin.v30.SequencerAdministrationService/OnboardingStateV2": 1
   }
-  admin-api.limits.pending {
+  admin-api.limits.active {
     "com.digitalasset.canton.topology.admin.v30.TopologyManagerReadService/GenesisState": 1,
     "com.digitalasset.canton.topology.admin.v30.TopologyManagerReadService/GenesisStateV2": 1,
     "com.digitalasset.canton.topology.admin.v30.TopologyManagerReadService/ExportTopologySnapshot": 1,

apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/plugins/UpdateHistorySanityCheckPlugin.scala

@@ -18,6 +18,7 @@ import com.digitalasset.canton.data.CantonTimestamp
 import com.digitalasset.canton.integration.EnvironmentSetupPlugin
 import com.digitalasset.canton.logging.SuppressingLogger
 import com.digitalasset.canton.tracing.TraceContext
+import org.lfdecentralizedtrust.splice.codegen.java.splice.dsorules.DsoRules
 import org.lfdecentralizedtrust.splice.store.UpdateHistory.BackfillingState
 import org.scalatest.{Inspectors, LoneElement}
 import org.scalatest.concurrent.Eventually
@@ -76,9 +77,27 @@ class UpdateHistorySanityCheckPlugin(
           // This flag should have the same value on all scans
           if (initializedScans.exists(_.config.updateHistoryBackfillEnabled)) {
             initializedScans.foreach(waitUntilBackfillingComplete)
-            compareHistories(initializedScans)
+            val (founders, others) = initializedScans.partition(_.config.isFirstSv)
+            val founder = founders.loneElement
+            val dsoRules =
+              DsoRules.fromJson(founder.getDsoInfo().dsoRules.contract.payload.noSpaces)
+            val (scansInDsoRules, scansNotInDsoRules) = others.partition { otherScan =>
+              val svPartyId = otherScan.getDsoInfo().svPartyId
+              dsoRules.svs.containsKey(svPartyId)
+            }
+            scansNotInDsoRules.foreach { notInDso =>
+              // the SV will not see transactions to the DSO, but will still see transactions involving itself,
+              // so comparison of history & snapshots will be broken if something like this happens:
+              //  - U1: DSO-only
+              //  - U2: Involves SV
+              // founder sees U1, U2 but otherScan only U2
+              logger.info(
+                s"The SV party of Scan ${notInDso.name} (partyId=${notInDso.getDsoInfo().svPartyId}) is not in DsoRules. Ignoring."
+              )
+            }
+            compareHistories(founder, scansInDsoRules)
             if (!skipAcsSnapshotChecks) {
-              compareSnapshots(initializedScans)
+              compareSnapshots(founder, scansInDsoRules)
             }
             initializedScans.foreach(checkScanTxLogScript)
           } else {
@@ -126,10 +145,9 @@ class UpdateHistorySanityCheckPlugin(
   }
 
   private def compareHistories(
-      scans: Seq[ScanAppBackendReference]
+      founder: ScanAppBackendReference,
+      others: Seq[ScanAppBackendReference],
   ): Unit = {
-    val (founders, others) = scans.partition(_.config.isFirstSv)
-    val founder = founders.loneElement
     val founderHistory = paginateHistory(founder, None, Chain.empty).toVector
     forAll(others) { otherScan =>
       withClue(s"Comparing ${otherScan.name} to ${founder.name}") {
@@ -266,9 +284,10 @@ class UpdateHistorySanityCheckPlugin(
     }
   }
 
-  private def compareSnapshots(scans: Seq[ScanAppBackendReference]) = {
-    val (founders, others) = scans.partition(_.config.isFirstSv)
-    val founder = founders.loneElement
+  private def compareSnapshots(
+      founder: ScanAppBackendReference,
+      others: Seq[ScanAppBackendReference],
+  ) = {
     val founderSnapshots = getAllSnapshots(founder, CantonTimestamp.MaxValue, Nil)
     forAll(others) { otherScan =>
       withClue(s"Comparing ${otherScan.name} to ${founder.name}") {

apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/DecentralizedSynchronizerMigrationIntegrationTest.scala

@@ -435,9 +435,11 @@ class DecentralizedSynchronizerMigrationIntegrationTest
         .getExternalPartyBalance(externalParty)
         .totalUnlockedCoin shouldBe "0.0000000000"
       walletClient.transferPreapprovalSend(externalParty, 40.0, UUID.randomUUID.toString)
-      validatorBackend
-        .getExternalPartyBalance(externalParty)
-        .totalUnlockedCoin shouldBe "40.0000000000"
+      eventually() {
+        validatorBackend
+          .getExternalPartyBalance(externalParty)
+          .totalUnlockedCoin shouldBe "40.0000000000"
+      }
       onboarding
     }
 

apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/DockerComposeValidatorFrontendIntegrationTest.scala

@@ -335,13 +335,20 @@ class DockerComposeValidatorFrontendIntegrationTest
       withFrontEnd("frontend") { implicit webDriver =>
         val validatorUserPassword = sys.env(s"COMPOSE_VALIDATOR_WEB_UI_PASSWORD")
         eventuallySucceeds()(go to s"http://wallet.localhost")
-        completeAuth0LoginWithAuthorization(
-          "http://wallet.localhost",
-          "[email protected]",
-          validatorUserPassword,
-          () => seleniumText(find(id("logged-in-user"))) should startWith(partyHint),
+
+        actAndCheck()(
+          s"Validator login to wallet via auth0",
+          completeAuth0LoginWithAuthorization(
+            "http://wallet.localhost",
+            "[email protected]",
+            validatorUserPassword,
+            () => seleniumText(find(id("logged-in-user"))) should startWith(partyHint),
+          ),
+        )(
+          "User is already logged in, and sees their balance",
+          _ => userLoggedInAndHasBalance("da-ComposeValidator-1::", adminTap),
         )
-        userLoggedInAndHasBalance("da-ComposeValidator-1::", adminTap)
+
         completeAuth0LoginWithAuthorization(
           "http://ans.localhost",
           "[email protected]",

apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/ManualStartIntegrationTest.scala

@@ -15,7 +15,7 @@ import com.digitalasset.canton.topology.{
   SequencerId,
   UniqueIdentifier,
 }
-import org.lfdecentralizedtrust.splice.config.{ConfigTransforms, SpliceBackendConfig}
+import org.lfdecentralizedtrust.splice.config.{ConfigTransforms, PruningConfig, SpliceBackendConfig}
 import org.lfdecentralizedtrust.splice.console.AppBackendReference
 import org.lfdecentralizedtrust.splice.environment.*
 import org.lfdecentralizedtrust.splice.integration.EnvironmentDefinition
@@ -25,7 +25,6 @@ import org.lfdecentralizedtrust.splice.integration.tests.SpliceTests.{
 }
 import org.lfdecentralizedtrust.splice.sv.config.SvAppBackendConfig
 import org.lfdecentralizedtrust.splice.util.{StandaloneCanton, TriggerTestUtil, WalletTestUtil}
-import org.lfdecentralizedtrust.splice.validator.config.ParticipantPruningConfig
 
 import java.util.UUID
 import scala.concurrent.duration.DurationInt
@@ -63,7 +62,7 @@ class ManualStartIntegrationTest
               aliceValidatorConfig.copy(
                 domains = withoutExtraDomains,
                 participantPruningSchedule = Some(
-                  ParticipantPruningConfig(
+                  PruningConfig(
                     "0 0 * * * ?",
                     PositiveDurationSeconds.tryFromDuration(1.hours),
                     PositiveDurationSeconds.tryFromDuration(30.hours),
@@ -169,7 +168,9 @@ class ManualStartIntegrationTest
           )
         }
 
-        clue("SV1 and SV2 have configured amplification on the mediator sequencer connection") {
+        clue(
+          "SV1 and SV2 have configured amplification and pruning on the mediator sequencer connection"
+        ) {
           Seq(
             mediatorAdminConnection("sv1", sv1Backend.config),
             mediatorAdminConnection("sv2", sv2Backend.config),
@@ -182,6 +183,11 @@ class ManualStartIntegrationTest
             sequencerConnections.connections.size shouldBe 1
             sequencerConnections.sequencerTrustThreshold shouldBe PositiveInt.tryCreate(1)
             sequencerConnections.submissionRequestAmplification shouldBe SvAppBackendConfig.DefaultMediatorSequencerRequestAmplification
+            mediatorConnection.getPruningSchedule().futureValue.value shouldBe PruningSchedule(
+              "0 /10 * * * ?",
+              PositiveDurationSeconds.ofMinutes(5),
+              PositiveDurationSeconds.ofDays(30),
+            )
             // otherwise we get log warnings
             mediatorConnection.close()
           }