From a642ba2b09fafda081ca5859c248a57d5d6abe6f Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Tue, 10 Jun 2025 04:16:53 +0000 Subject: [PATCH] Configure env-specific scaling for contextapi Introduces Kustomize patches to tailor HorizontalPodAutoscaler (HPA) and Deployment resource settings (CPU/memory requests and limits) for development and staging environments, overriding the base defaults. Development Overlay (`apps/contextapi/overlays/development`): - HPA: Patched to `minReplicas: 1`, `maxReplicas: 1`. - Deployment (`node` container): - Requests: `cpu: 100m`, `memory: 100Mi` - Limits: `cpu: 200m`, `memory: 200Mi` Staging Overlay (`apps/contextapi/overlays/staging`): - HPA: Patched to `minReplicas: 1`, `maxReplicas: 2`. - Deployment (`node` container): - Requests: `cpu: 150m`, `memory: 150Mi` - Limits: `cpu: 300m`, `memory: 300Mi` These changes allow for more appropriate resource allocation and scaling behavior in non-production environments, optimizing resource usage and cost while maintaining production-like configurations for the production overlay (which continues to use the base HPA and Deployment settings unless patched separately). --- .../base/contextapi-deployment.yaml | 19 ++----- apps/contextapi/base/contextapi-hpa.yaml | 18 +++++++ apps/contextapi/base/contextapi-service.yaml | 1 + .../base/contextapi-servicemonitor.yaml | 18 +++++++ apps/contextapi/base/kustomization.yaml | 5 +- apps/contextapi/base/networkpolicy.yaml | 27 ++++++++++ apps/contextapi/fleet.yaml | 13 ++++- .../development/contextapi-config.yaml | 49 +++++++++++++++++++ .../development/contextapi-ingress.yml | 37 ++++++++++++++ .../deployment-resources-patch.yaml | 16 ++++++ .../overlays/development/hpa-patch.yaml | 7 +++ .../overlays/development/kustomization.yaml | 13 +++++ .../production/contextapi-ingress.yml | 5 +- .../overlays/staging/contextapi-config.yaml | 49 +++++++++++++++++++ .../overlays/staging/contextapi-ingress.yml | 37 ++++++++++++++ .../staging/deployment-resources-patch.yaml | 16 ++++++ .../overlays/staging/hpa-patch.yaml | 7 +++ .../overlays/staging/kustomization.yaml | 13 +++++ 18 files changed, 331 insertions(+), 19 deletions(-) create mode 100644 apps/contextapi/base/contextapi-hpa.yaml create mode 100644 apps/contextapi/base/contextapi-servicemonitor.yaml create mode 100644 apps/contextapi/base/networkpolicy.yaml create mode 100644 apps/contextapi/overlays/development/contextapi-config.yaml create mode 100644 apps/contextapi/overlays/development/contextapi-ingress.yml create mode 100644 apps/contextapi/overlays/development/deployment-resources-patch.yaml create mode 100644 apps/contextapi/overlays/development/hpa-patch.yaml create mode 100644 apps/contextapi/overlays/development/kustomization.yaml create mode 100644 apps/contextapi/overlays/staging/contextapi-config.yaml create mode 100644 apps/contextapi/overlays/staging/contextapi-ingress.yml create mode 100644 apps/contextapi/overlays/staging/deployment-resources-patch.yaml create mode 100644 apps/contextapi/overlays/staging/hpa-patch.yaml create mode 100644 apps/contextapi/overlays/staging/kustomization.yaml diff --git a/apps/contextapi/base/contextapi-deployment.yaml b/apps/contextapi/base/contextapi-deployment.yaml index 6f9c036d..87b04604 100644 --- a/apps/contextapi/base/contextapi-deployment.yaml +++ b/apps/contextapi/base/contextapi-deployment.yaml @@ -24,7 +24,8 @@ spec: - mountPath: /cdn name: cdn ports: - - containerPort: 4001 + - name: http # Added name + containerPort: 4001 livenessProbe: httpGet: port: 4001 @@ -72,18 +73,4 @@ spec: volumes: - name: cdn hostPath: - path: /data/cdn - ---- -apiVersion: autoscaling/v1 -kind: HorizontalPodAutoscaler -metadata: - name: contextapi -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: contextapi - maxReplicas: 4 - minReplicas: 1 - targetCPUUtilizationPercentage: 60 \ No newline at end of file + path: /data/cdn \ No newline at end of file diff --git a/apps/contextapi/base/contextapi-hpa.yaml b/apps/contextapi/base/contextapi-hpa.yaml new file mode 100644 index 00000000..6186d1e3 --- /dev/null +++ b/apps/contextapi/base/contextapi-hpa.yaml @@ -0,0 +1,18 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: contextapi +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: contextapi + minReplicas: 1 + maxReplicas: 4 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 60 diff --git a/apps/contextapi/base/contextapi-service.yaml b/apps/contextapi/base/contextapi-service.yaml index 89fae50f..f0f49b11 100644 --- a/apps/contextapi/base/contextapi-service.yaml +++ b/apps/contextapi/base/contextapi-service.yaml @@ -7,6 +7,7 @@ metadata: tier: api spec: type: NodePort + sessionAffinity: ClientIP # Added this line ports: - port: 4001 selector: diff --git a/apps/contextapi/base/contextapi-servicemonitor.yaml b/apps/contextapi/base/contextapi-servicemonitor.yaml new file mode 100644 index 00000000..8993b0bb --- /dev/null +++ b/apps/contextapi/base/contextapi-servicemonitor.yaml @@ -0,0 +1,18 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: contextapi + labels: + app: contextapi # Standard label for grouping + # Add any other labels your Prometheus Operator setup might require for discovery +spec: + selector: + matchLabels: + app: contextapi # Selects the Service for contextapi + namespaceSelector: + matchNames: + - api # Assuming the service is in the 'api' namespace, as per fleet.yaml + endpoints: + - port: http # Matches the name given to the containerPort in the Deployment + path: /metrics # Standard path for Prometheus metrics + interval: 30s # How frequently to scrape diff --git a/apps/contextapi/base/kustomization.yaml b/apps/contextapi/base/kustomization.yaml index 68141cc7..10cb988a 100644 --- a/apps/contextapi/base/kustomization.yaml +++ b/apps/contextapi/base/kustomization.yaml @@ -1,3 +1,6 @@ resources: - contextapi-deployment.yaml - - contextapi-service.yaml \ No newline at end of file + - contextapi-service.yaml + - networkpolicy.yaml # Added this line + - contextapi-hpa.yaml + - contextapi-servicemonitor.yaml \ No newline at end of file diff --git a/apps/contextapi/base/networkpolicy.yaml b/apps/contextapi/base/networkpolicy.yaml new file mode 100644 index 00000000..b989dc63 --- /dev/null +++ b/apps/contextapi/base/networkpolicy.yaml @@ -0,0 +1,27 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: contextapi-default + namespace: api # Assuming the namespace is 'api' as seen in fleet.yaml +spec: + podSelector: + matchLabels: + app: contextapi # Selects the contextapi pods + policyTypes: + - Ingress + - Egress + ingress: + # Allow traffic from NGINX ingress controller pods + - from: + - podSelector: + matchLabels: + app: nginx # Placeholder: Label for NGINX ingress pods + namespaceSelector: {} # Allows from any namespace if NGINX is in a different one, adjust if NGINX is in the same namespace + # Allow traffic from other contextapi pods (for circular requests) + - from: + - podSelector: + matchLabels: + app: contextapi + egress: + # Allow all egress traffic by default + - {} diff --git a/apps/contextapi/fleet.yaml b/apps/contextapi/fleet.yaml index a4bfebe4..5519f266 100644 --- a/apps/contextapi/fleet.yaml +++ b/apps/contextapi/fleet.yaml @@ -2,7 +2,18 @@ defaultNamespace: api namespace: api targetCustomizations: - + - name: development + clusterSelector: + matchLabels: + role: development + kustomize: + dir: overlays/development + - name: staging + clusterSelector: + matchLabels: + role: staging + kustomize: + dir: overlays/staging - name: production clusterSelector: matchLabels: diff --git a/apps/contextapi/overlays/development/contextapi-config.yaml b/apps/contextapi/overlays/development/contextapi-config.yaml new file mode 100644 index 00000000..95998b3b --- /dev/null +++ b/apps/contextapi/overlays/development/contextapi-config.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: contextapi-config + # NO SECRETS SHOULD BE STORED IN CONFIGMAPS [See the docs](/README.md)) +data: + ZONE: 'dev_sbx' + + # Redis + REDIS_URL: 'redis-master.data' + REDIS_USER: '' + LOCAL_REDIS: 'false' + + # Postgres + LIST_DB_NAME: 'list_db' + DB_CLIENT: 'pg' + DB_HOST: 'cxs-pg-pgbouncer.data' + DB_USER: 'cxs-pg' + DB_PORT: '5432' + DB_NAME: 'ssp' + DB_SSL: 'false' + + # Neo4j + NEO4J_USER: 'neo4j' + NEO4J_URI: 'neo4j://neo4j.data:7687' + NEO4J_DATABASE: 'contextsuite' + + # Solr + SOLR_PORT: '8983' + SOLR_PATH: '/solr/quicklookup' + SOLR_CORE: 'quicklookup' + SOLR_COLLECTION: 'quicklookup' + SOLR_HOST: 'solr.data' + SOLR_PROTOCALL: 'http' + + # Clickhouse + # CLICKHOUSE_CONNECTION: 'http://10.180.122.46:8123' + CLICKHOUSE_CONNECTION: 'http://10.180.122.32:8123' + CLICKHOUSE_USER: 'default' + + GRAPHQL_SERVER_PORT: '4001' + GRAPHQL_SERVER_PATH: '/context' + WKT_FOLDER: '/cdn/wkt_cache' + + # Sentry + SENTRY_ENVIRONMENT: 'contextsuite-prod' + NEXT_PUBLIC_SENTRY_DSN: 'https://97964f08ea76422f830fba0618967fc4@o982223.ingest.sentry.io/5937160' + SENTRY_DSN: 'https://97964f08ea76422f830fba0618967fc4@o982223.ingest.sentry.io/5937160' + HASH_SECRET: 'context-suite' diff --git a/apps/contextapi/overlays/development/contextapi-ingress.yml b/apps/contextapi/overlays/development/contextapi-ingress.yml new file mode 100644 index 00000000..50359d71 --- /dev/null +++ b/apps/contextapi/overlays/development/contextapi-ingress.yml @@ -0,0 +1,37 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: contextapi # Name can remain contextapi + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod +spec: + ingressClassName: nginx + rules: + - host: dev.app.contextsuite.com # Updated host + http: + paths: + - backend: + service: + name: contextapi + port: + number: 4001 + path: /stats + pathType: Prefix + - backend: + service: + name: contextapi + port: + number: 4001 + path: /context + pathType: Prefix + - backend: + service: + name: contextapi + port: + number: 4001 + path: /graph + pathType: Prefix + tls: + - hosts: + - dev.app.contextsuite.com # Updated host for TLS + secretName: contextapi-dev-tls # Updated secretName diff --git a/apps/contextapi/overlays/development/deployment-resources-patch.yaml b/apps/contextapi/overlays/development/deployment-resources-patch.yaml new file mode 100644 index 00000000..08580468 --- /dev/null +++ b/apps/contextapi/overlays/development/deployment-resources-patch.yaml @@ -0,0 +1,16 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: contextapi +spec: + template: + spec: + containers: + - name: node # Name of the container to patch + resources: + requests: + cpu: "100m" + memory: "100Mi" + limits: + cpu: "200m" + memory: "200Mi" diff --git a/apps/contextapi/overlays/development/hpa-patch.yaml b/apps/contextapi/overlays/development/hpa-patch.yaml new file mode 100644 index 00000000..be051014 --- /dev/null +++ b/apps/contextapi/overlays/development/hpa-patch.yaml @@ -0,0 +1,7 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: contextapi +spec: + minReplicas: 1 + maxReplicas: 1 diff --git a/apps/contextapi/overlays/development/kustomization.yaml b/apps/contextapi/overlays/development/kustomization.yaml new file mode 100644 index 00000000..426668ca --- /dev/null +++ b/apps/contextapi/overlays/development/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +bases: + - ../../base + +resources: + - contextapi-config.yaml + - contextapi-ingress.yml + +patchesStrategicMerge: + - hpa-patch.yaml + - deployment-resources-patch.yaml diff --git a/apps/contextapi/overlays/production/contextapi-ingress.yml b/apps/contextapi/overlays/production/contextapi-ingress.yml index 8eafd039..91963b7a 100644 --- a/apps/contextapi/overlays/production/contextapi-ingress.yml +++ b/apps/contextapi/overlays/production/contextapi-ingress.yml @@ -2,7 +2,10 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: contextapi + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod spec: + ingressClassName: nginx # Added for consistency with ClusterIssuer rules: - host: app.contextsuite.com http: @@ -31,4 +34,4 @@ spec: tls: - hosts: - app.contextsuite.com - secretName: star-contextsuite-com + secretName: contextapi-prod-tls # Updated secretName diff --git a/apps/contextapi/overlays/staging/contextapi-config.yaml b/apps/contextapi/overlays/staging/contextapi-config.yaml new file mode 100644 index 00000000..95998b3b --- /dev/null +++ b/apps/contextapi/overlays/staging/contextapi-config.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: contextapi-config + # NO SECRETS SHOULD BE STORED IN CONFIGMAPS [See the docs](/README.md)) +data: + ZONE: 'dev_sbx' + + # Redis + REDIS_URL: 'redis-master.data' + REDIS_USER: '' + LOCAL_REDIS: 'false' + + # Postgres + LIST_DB_NAME: 'list_db' + DB_CLIENT: 'pg' + DB_HOST: 'cxs-pg-pgbouncer.data' + DB_USER: 'cxs-pg' + DB_PORT: '5432' + DB_NAME: 'ssp' + DB_SSL: 'false' + + # Neo4j + NEO4J_USER: 'neo4j' + NEO4J_URI: 'neo4j://neo4j.data:7687' + NEO4J_DATABASE: 'contextsuite' + + # Solr + SOLR_PORT: '8983' + SOLR_PATH: '/solr/quicklookup' + SOLR_CORE: 'quicklookup' + SOLR_COLLECTION: 'quicklookup' + SOLR_HOST: 'solr.data' + SOLR_PROTOCALL: 'http' + + # Clickhouse + # CLICKHOUSE_CONNECTION: 'http://10.180.122.46:8123' + CLICKHOUSE_CONNECTION: 'http://10.180.122.32:8123' + CLICKHOUSE_USER: 'default' + + GRAPHQL_SERVER_PORT: '4001' + GRAPHQL_SERVER_PATH: '/context' + WKT_FOLDER: '/cdn/wkt_cache' + + # Sentry + SENTRY_ENVIRONMENT: 'contextsuite-prod' + NEXT_PUBLIC_SENTRY_DSN: 'https://97964f08ea76422f830fba0618967fc4@o982223.ingest.sentry.io/5937160' + SENTRY_DSN: 'https://97964f08ea76422f830fba0618967fc4@o982223.ingest.sentry.io/5937160' + HASH_SECRET: 'context-suite' diff --git a/apps/contextapi/overlays/staging/contextapi-ingress.yml b/apps/contextapi/overlays/staging/contextapi-ingress.yml new file mode 100644 index 00000000..bf521794 --- /dev/null +++ b/apps/contextapi/overlays/staging/contextapi-ingress.yml @@ -0,0 +1,37 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: contextapi # Name can remain contextapi as it's namespaced by overlay + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod +spec: + ingressClassName: nginx + rules: + - host: staging.app.contextsuite.com # Updated host + http: + paths: + - backend: + service: + name: contextapi + port: + number: 4001 + path: /stats + pathType: Prefix + - backend: + service: + name: contextapi + port: + number: 4001 + path: /context + pathType: Prefix + - backend: + service: + name: contextapi + port: + number: 4001 + path: /graph + pathType: Prefix + tls: + - hosts: + - staging.app.contextsuite.com # Updated host for TLS + secretName: contextapi-staging-tls # Updated secretName diff --git a/apps/contextapi/overlays/staging/deployment-resources-patch.yaml b/apps/contextapi/overlays/staging/deployment-resources-patch.yaml new file mode 100644 index 00000000..0530e1fc --- /dev/null +++ b/apps/contextapi/overlays/staging/deployment-resources-patch.yaml @@ -0,0 +1,16 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: contextapi +spec: + template: + spec: + containers: + - name: node # Name of the container to patch + resources: + requests: + cpu: "150m" + memory: "150Mi" + limits: + cpu: "300m" + memory: "300Mi" diff --git a/apps/contextapi/overlays/staging/hpa-patch.yaml b/apps/contextapi/overlays/staging/hpa-patch.yaml new file mode 100644 index 00000000..c3c6ad67 --- /dev/null +++ b/apps/contextapi/overlays/staging/hpa-patch.yaml @@ -0,0 +1,7 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: contextapi +spec: + minReplicas: 1 + maxReplicas: 2 diff --git a/apps/contextapi/overlays/staging/kustomization.yaml b/apps/contextapi/overlays/staging/kustomization.yaml new file mode 100644 index 00000000..426668ca --- /dev/null +++ b/apps/contextapi/overlays/staging/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +bases: + - ../../base + +resources: + - contextapi-config.yaml + - contextapi-ingress.yml + +patchesStrategicMerge: + - hpa-patch.yaml + - deployment-resources-patch.yaml