From cdf5376627d414f14eefa69f2bc2e80ea08074b4 Mon Sep 17 00:00:00 2001 From: Jacek Nykis Date: Thu, 30 Jan 2025 14:12:05 +0000 Subject: [PATCH 1/4] Update config to remove need for multiple Ingress objects This code removes supercluster deployed Ingress and it modified the code to allow the `SimplePayment` mission to pass with a catch-all nginx proxy. Proxy config I used is: ``` server { listen 80 default_server; server_name _; resolver 10.96.0.10 ipv6=off; location ~ ^/(.+)-([0-9]+)/core$ { proxy_pass http://$1-$2.$1.stellar-supercluster.svc.cluster.local:11626/; } location ~ ^/(.+)-([0-9]+)/core/(.*)$ { proxy_pass http://$1-$2.$1.stellar-supercluster.svc.cluster.local:11626/$3$is_args$args; } location ~ ^/(.+)-([0-9]+)/history$ { proxy_pass http://$1-$2.$1.stellar-supercluster.svc.cluster.local:80/; } location ~ ^/(.+)-([0-9]+)/history/(.*)$ { proxy_pass http://$1-$2.$1.stellar-supercluster.svc.cluster.local:80/$3; } } ``` To allow me to target pods directly I had to modify serviceName in the StatefulSet --- src/FSLibrary.Tests/Tests.fs | 2 +- src/FSLibrary/StellarNetworkCfg.fs | 2 +- src/FSLibrary/StellarSupercluster.fs | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index c3992890..eae2c06f 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -137,7 +137,7 @@ type Tests(output: ITestOutputHelper) = let peer1DNS = (nCfg.PeerDnsName coreSet 1).StringName let peer2DNS = (nCfg.PeerDnsName coreSet 2).StringName let nonceStr = nCfg.networkNonce.ToString() - let domain = nonceStr + "-stellar-core." + ctx.namespaceProperty + ".svc.cluster.local" + let domain = nonceStr + "-sts-core." + ctx.namespaceProperty + ".svc.cluster.local" Assert.Equal(nonceStr + "-sts-test-0." + domain, peer0DNS) Assert.Equal(nonceStr + "-sts-test-1." + domain, peer1DNS) Assert.Equal(nonceStr + "-sts-test-2." + domain, peer2DNS) diff --git a/src/FSLibrary/StellarNetworkCfg.fs b/src/FSLibrary/StellarNetworkCfg.fs index 3cb7370f..a5ee5680 100644 --- a/src/FSLibrary/StellarNetworkCfg.fs +++ b/src/FSLibrary/StellarNetworkCfg.fs @@ -91,7 +91,7 @@ type NetworkCfg = member self.PeerShortName (cs: CoreSet) (n: int) : PeerShortName = PeerShortName(sprintf "%s-%d" cs.name.StringName n) - member self.ServiceName : string = sprintf "%s-stellar-core" self.Nonce + member self.ServiceName : string = sprintf "%s-sts-core" self.Nonce member self.IngressName : string = sprintf "%s-stellar-core-ingress" self.Nonce diff --git a/src/FSLibrary/StellarSupercluster.fs b/src/FSLibrary/StellarSupercluster.fs index 3176be18..cc6c6941 100644 --- a/src/FSLibrary/StellarSupercluster.fs +++ b/src/FSLibrary/StellarSupercluster.fs @@ -145,10 +145,10 @@ type Kubernetes with if not (List.isEmpty statefulSets) then let ing = nCfg.ToIngress() - LogInfo "Creating Ingress %s" ing.Metadata.Name + LogInfo "Skipping creation of Ingress %s" ing.Metadata.Name ApiRateLimit.sleepUntilNextRateLimitedApiCallTime (rps) - let ingress = self.CreateNamespacedIngress(namespaceParameter = nsStr, body = ing) - namespaceContent.Add(ingress) + //let ingress = self.CreateNamespacedIngress(namespaceParameter = nsStr, body = ing) + //namespaceContent.Add(ingress) let formation = new StellarFormation( From 858862b16a0c4c6958f2e5ffff2469b610822e96 Mon Sep 17 00:00:00 2001 From: Jay Geng Date: Thu, 13 Feb 2025 00:12:04 -0500 Subject: [PATCH 2/4] unify per-pod service name with statefulset name --- src/FSLibrary/StellarFormation.fs | 2 +- src/FSLibrary/StellarKubeSpecs.fs | 4 ++-- src/FSLibrary/StellarNetworkCfg.fs | 6 ++++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/FSLibrary/StellarFormation.fs b/src/FSLibrary/StellarFormation.fs index f38bee8f..c9198e23 100644 --- a/src/FSLibrary/StellarFormation.fs +++ b/src/FSLibrary/StellarFormation.fs @@ -128,6 +128,6 @@ type StellarFormation override self.Finalize() = self.Cleanup(false) override self.ToString() : string = - let name = networkCfg.ServiceName + let name = networkCfg.HeadlessServiceName let ns = networkCfg.NamespaceProperty sprintf "%s/%s" ns name diff --git a/src/FSLibrary/StellarKubeSpecs.fs b/src/FSLibrary/StellarKubeSpecs.fs index 067f2bd1..d63f4ae5 100644 --- a/src/FSLibrary/StellarKubeSpecs.fs +++ b/src/FSLibrary/StellarKubeSpecs.fs @@ -806,7 +806,7 @@ type NetworkCfg with // requires that you install the DNS server component on your k8s cluster. member self.ToService() : V1Service = let serviceSpec = V1ServiceSpec(clusterIP = "None", selector = CfgVal.labels) - V1Service(spec = serviceSpec, metadata = self.NamespacedMeta self.ServiceName) + V1Service(spec = serviceSpec, metadata = self.NamespacedMeta self.HeadlessServiceName) // Returns a StatefulSet object that will build stellar-core Pods named @@ -815,7 +815,7 @@ type NetworkCfg with let statefulSetSpec = V1StatefulSetSpec( selector = V1LabelSelector(matchLabels = CfgVal.labels), - serviceName = self.ServiceName, + serviceName = self.HeadlessServiceName, podManagementPolicy = "Parallel", template = self.ToPodTemplateSpec coreSet, replicas = System.Nullable(coreSet.CurrentCount) diff --git a/src/FSLibrary/StellarNetworkCfg.fs b/src/FSLibrary/StellarNetworkCfg.fs index a5ee5680..3114a33e 100644 --- a/src/FSLibrary/StellarNetworkCfg.fs +++ b/src/FSLibrary/StellarNetworkCfg.fs @@ -91,7 +91,9 @@ type NetworkCfg = member self.PeerShortName (cs: CoreSet) (n: int) : PeerShortName = PeerShortName(sprintf "%s-%d" cs.name.StringName n) - member self.ServiceName : string = sprintf "%s-sts-core" self.Nonce + member self.HeadlessServiceName : string = sprintf "%s-stellar-core" self.Nonce + + member self.ServiceName(cs: CoreSet) : string = self.StatefulSetName(cs).StringName member self.IngressName : string = sprintf "%s-stellar-core-ingress" self.Nonce @@ -107,7 +109,7 @@ type NetworkCfg = member self.PeerDnsName (cs: CoreSet) (n: int) : PeerDnsName = let s = - sprintf "%s.%s.%s.svc.cluster.local" (self.PodName cs n).StringName self.ServiceName self.NamespaceProperty + sprintf "%s.%s.%s.svc.cluster.local" (self.PodName cs n).StringName (self.ServiceName cs) self.NamespaceProperty PeerDnsName s From d3e0b6b691326899969db9bd93a456895a86022f Mon Sep 17 00:00:00 2001 From: Jay Geng Date: Tue, 25 Feb 2025 12:53:00 -0500 Subject: [PATCH 3/4] make the headless service name matching the sts name --- src/FSLibrary/StellarFormation.fs | 1 + src/FSLibrary/StellarKubeSpecs.fs | 7 ++++--- src/FSLibrary/StellarSupercluster.fs | 15 +++++++++++---- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/FSLibrary/StellarFormation.fs b/src/FSLibrary/StellarFormation.fs index c9198e23..8cd2186d 100644 --- a/src/FSLibrary/StellarFormation.fs +++ b/src/FSLibrary/StellarFormation.fs @@ -128,6 +128,7 @@ type StellarFormation override self.Finalize() = self.Cleanup(false) override self.ToString() : string = + // TODO: this one is just for display, don't care for now let name = networkCfg.HeadlessServiceName let ns = networkCfg.NamespaceProperty sprintf "%s/%s" ns name diff --git a/src/FSLibrary/StellarKubeSpecs.fs b/src/FSLibrary/StellarKubeSpecs.fs index d63f4ae5..4165135e 100644 --- a/src/FSLibrary/StellarKubeSpecs.fs +++ b/src/FSLibrary/StellarKubeSpecs.fs @@ -804,9 +804,10 @@ type NetworkCfg with // then use to connect the peers to one another in their config files, and // hook the per-Pod Services and Ingress up to). Getting all this to work // requires that you install the DNS server component on your k8s cluster. - member self.ToService() : V1Service = + member self.ToService(coreSet: CoreSet) : V1Service = let serviceSpec = V1ServiceSpec(clusterIP = "None", selector = CfgVal.labels) - V1Service(spec = serviceSpec, metadata = self.NamespacedMeta self.HeadlessServiceName) + let svcName =self.ServiceName(coreSet) + V1Service(spec = serviceSpec, metadata = self.NamespacedMeta svcName) // Returns a StatefulSet object that will build stellar-core Pods named @@ -815,7 +816,7 @@ type NetworkCfg with let statefulSetSpec = V1StatefulSetSpec( selector = V1LabelSelector(matchLabels = CfgVal.labels), - serviceName = self.HeadlessServiceName, + serviceName = self.ServiceName(coreSet) , //self.HeadlessServiceName, podManagementPolicy = "Parallel", template = self.ToPodTemplateSpec coreSet, replicas = System.Nullable(coreSet.CurrentCount) diff --git a/src/FSLibrary/StellarSupercluster.fs b/src/FSLibrary/StellarSupercluster.fs index cc6c6941..dff1d6f9 100644 --- a/src/FSLibrary/StellarSupercluster.fs +++ b/src/FSLibrary/StellarSupercluster.fs @@ -115,10 +115,17 @@ type Kubernetes with let rps = nCfg.missionContext.apiRateLimit try - let svc = nCfg.ToService() - LogInfo "Creating Service %s" svc.Metadata.Name - ApiRateLimit.sleepUntilNextRateLimitedApiCallTime (rps) - namespaceContent.Add(self.CreateNamespacedService(body = svc, namespaceParameter = nsStr)) + + let makePerStsServices coreSet = + let svc = nCfg.ToService(coreSet) + LogInfo "Creating Service %s" svc.Metadata.Name + ApiRateLimit.sleepUntilNextRateLimitedApiCallTime (rps) + self.CreateNamespacedService(body = svc, namespaceParameter = nsStr) + + let headlessSvcs = List.map makePerStsServices nCfg.CoreSetList + + for svc in headlessSvcs do + namespaceContent.Add(svc) for cm in nCfg.ToConfigMaps() do LogInfo "Creating ConfigMap %s" cm.Metadata.Name From 9f49035a9b6f034527a87d53f81815aba9214252 Mon Sep 17 00:00:00 2001 From: Jay Geng Date: Tue, 25 Feb 2025 13:07:51 -0500 Subject: [PATCH 4/4] Add a comment --- src/FSLibrary/StellarSupercluster.fs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/FSLibrary/StellarSupercluster.fs b/src/FSLibrary/StellarSupercluster.fs index dff1d6f9..4463f857 100644 --- a/src/FSLibrary/StellarSupercluster.fs +++ b/src/FSLibrary/StellarSupercluster.fs @@ -116,6 +116,9 @@ type Kubernetes with try + // NB: I *think* this is the correct change for the headless service name. + // Previously it only creates a single service with a fixed name, despite potentially there are more than one coreSet. + // The headless service is needed to register local DNS names for each of the Pod names in the StatefulSet, which implies one-per-StatefulSEt. let makePerStsServices coreSet = let svc = nCfg.ToService(coreSet) LogInfo "Creating Service %s" svc.Metadata.Name