fix(deployment): adjust the resources needed for otel-collector in each env (#6576)

Author: dotansimha

deployment/index.ts

@@ -72,10 +72,7 @@ const environment = prepareEnvironment({
   rootDns: new pulumi.Config('common').require('dnsZone'),
 });
 deploySentryEventsMonitor({ docker, environment, sentry });
-const observability = deployObservability({
-  envName,
-  tableSuffix: envName === 'prod' ? 'production' : envName,
-});
+const observability = deployObservability({ environment });
 const clickhouse = deployClickhouse();
 const postgres = deployPostgres();
 const redis = deployRedis({ environment });

deployment/services/observability.ts

@@ -1,19 +1,13 @@
 import * as pulumi from '@pulumi/pulumi';
 import { serviceLocalHost } from '../utils/local-endpoint';
 import { Observability as ObservabilityInstance } from '../utils/observability';
+import { Environment } from './environment';
 import { deployGrafana } from './grafana';
 
 // Change this to control OTEL tracing for usage service
 const enableTracingForUsageService = true;
 
-export function deployObservability(config: {
-  envName: string;
-  /**
-   * Suffix for the table names (production, staging, dev).
-   * It can't be envName as "prod" is not a valid table suffix.
-   */
-  tableSuffix: string;
-}) {
+export function deployObservability(config: { environment: Environment }) {
   const observabilityConfig = new pulumi.Config('observability');
 
   if (!observabilityConfig.getBoolean('enabled')) {
@@ -25,7 +19,7 @@ export function deployObservability(config: {
   const useLocal = observabilityConfig.getBoolean('local');
 
   const observability = new ObservabilityInstance(
-    config.envName,
+    config.environment,
     useLocal
       ? 'local'
       : {
@@ -53,12 +47,15 @@ export function deployObservability(config: {
     throw new Error('OTLP collector service is required for observability');
   }
 
+  const tableSuffix =
+    config.environment.envName === 'prod' ? 'production' : config.environment.envName;
+
   return {
     tracingEndpoint: serviceLocalHost(observabilityInstance.otlpCollectorService).apply(
       host => `http://${host}:4318/v1/traces`,
     ),
     observability: observabilityInstance,
-    grafana: useLocal ? undefined : deployGrafana(config.envName, config.tableSuffix),
+    grafana: useLocal ? undefined : deployGrafana(config.environment.envName, tableSuffix),
     enabled: true,
     enabledForUsageService: enableTracingForUsageService,
   };

deployment/utils/observability.ts

@@ -1,5 +1,6 @@
 import * as k8s from '@pulumi/kubernetes';
 import { interpolate, Output } from '@pulumi/pulumi';
+import { Environment } from '../services/environment';
 import { helmChart } from './helm';
 import { Values as OpenTelemetryCollectorValues } from './opentelemetry-collector.types';
 import { VectorValues } from './vector.types';
@@ -31,7 +32,7 @@ export const VECTOR_HELM_CHART = helmChart('https://helm.vector.dev', 'vector',
 
 export class Observability {
   constructor(
-    private envName: string,
+    private environment: Environment,
     private config: ObservabilityConfig,
   ) {}
 
@@ -92,7 +93,7 @@ export class Observability {
               labels: {
                 namespace: '{{`{{ kubernetes.pod_namespace }}`}}',
                 container_name: '{{`{{ kubernetes.container_name }}`}}',
-                env: this.envName,
+                env: this.environment.envName,
               },
               encoding: {
                 codec: 'text',
@@ -111,8 +112,8 @@ export class Observability {
       replicaCount: 1,
       resources: {
         limits: {
-          cpu: '512m',
-          memory: '1000Mi',
+          cpu: this.environment.isProduction ? '512m' : '150m',
+          memory: this.environment.isProduction ? '1000Mi' : '300Mi',
         },
       },
       podAnnotations: {