Skip to content

Commit 9ff66d0

Browse files
xDarksomexDarksome
authored
feat(infra): export prometheus metrics to s3 (#310)
1 parent 8752960 commit 9ff66d0

3 files changed

Lines changed: 205 additions & 109 deletions

File tree

‎infra/modules/node-operator/main.tf‎

Lines changed: 127 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,10 @@ variable "config" {
4444
cpu_cores = number
4545
memory = number
4646
disk = number
47+
48+
prom2parquet_image = optional(string)
49+
s3_bucket = optional(string)
50+
s3_metrics_prefix = optional(string)
4751
}))
4852

4953
grafana = optional(object({
@@ -68,6 +72,7 @@ variable "config" {
6872

6973
locals {
7074
create_ec2_instance_connect_endpoint = coalesce(var.config.create_ec2_instance_connect_endpoint, true)
75+
prometheus_s3_export = try(var.config.prometheus.prom2parquet_image, null) != null
7176
}
7277

7378
data "aws_region" "current" {}
@@ -134,22 +139,25 @@ module "db" {
134139

135140
public_ip = false
136141

137-
ports = [
138-
{ port = local.db_primary_rpc_server_port, protocol = "udp", internal = true },
139-
{ port = local.db_secondary_rpc_server_port, protocol = "udp", internal = true },
140-
{ port = local.db_metrics_server_port, protocol = "tcp", internal = true },
141-
]
142-
143-
environment = {
144-
PRIMARY_RPC_SERVER_PORT = tostring(local.db_primary_rpc_server_port)
145-
SECONDARY_RPC_SERVER_PORT = tostring(local.db_secondary_rpc_server_port)
146-
METRICS_SERVER_PORT = tostring(local.db_metrics_server_port)
147-
ROCKSDB_DIR = "/data"
148-
}
142+
containers = [{
143+
image = var.config.db.image
144+
ports = [
145+
{ port = local.db_primary_rpc_server_port, protocol = "udp", internal = true },
146+
{ port = local.db_secondary_rpc_server_port, protocol = "udp", internal = true },
147+
{ port = local.db_metrics_server_port, protocol = "tcp", internal = true },
148+
]
149+
150+
environment = {
151+
PRIMARY_RPC_SERVER_PORT = tostring(local.db_primary_rpc_server_port)
152+
SECONDARY_RPC_SERVER_PORT = tostring(local.db_secondary_rpc_server_port)
153+
METRICS_SERVER_PORT = tostring(local.db_metrics_server_port)
154+
ROCKSDB_DIR = "/data"
155+
}
149156

150-
secrets = {
151-
SECRET_KEY = module.secret["ed25519_secret_key"]
152-
}
157+
secrets = {
158+
SECRET_KEY = module.secret["ed25519_secret_key"]
159+
}
160+
}]
153161
})
154162
}
155163

@@ -170,37 +178,41 @@ module "node" {
170178

171179
public_ip = true
172180

173-
ports = [
174-
{ port = local.node_primary_rpc_server_port, protocol = "udp", internal = false },
175-
{ port = local.node_secondary_rpc_server_port, protocol = "udp", internal = false },
176-
{ port = local.node_metrics_server_port, protocol = "tcp", internal = true },
177-
]
178-
179-
environment = {
180-
PRIMARY_RPC_SERVER_PORT = tostring(local.node_primary_rpc_server_port)
181-
SECONDARY_RPC_SERVER_PORT = tostring(local.node_secondary_rpc_server_port)
182-
METRICS_SERVER_PORT = tostring(local.node_metrics_server_port)
183-
DATABASE_RPC_SERVER_ADDRESS = module.db.private_ip
184-
DATABASE_PEER_ID = local.peer_id
185-
DATABASE_PRIMARY_RPC_SERVER_PORT = tostring(local.db_primary_rpc_server_port)
186-
DATABASE_SECONDARY_RPC_SERVER_PORT = tostring(local.db_secondary_rpc_server_port)
187-
SMART_CONTRACT_ADDRESS = local.smart_contract_address
188-
}
181+
containers = [{
182+
image = var.config.nodes[count.index].image
183+
ports = [
184+
{ port = local.node_primary_rpc_server_port, protocol = "udp", internal = false },
185+
{ port = local.node_secondary_rpc_server_port, protocol = "udp", internal = false },
186+
{ port = local.node_metrics_server_port, protocol = "tcp", internal = true },
187+
]
188+
189+
environment = {
190+
PRIMARY_RPC_SERVER_PORT = tostring(local.node_primary_rpc_server_port)
191+
SECONDARY_RPC_SERVER_PORT = tostring(local.node_secondary_rpc_server_port)
192+
METRICS_SERVER_PORT = tostring(local.node_metrics_server_port)
193+
DATABASE_RPC_SERVER_ADDRESS = module.db.private_ip
194+
DATABASE_PEER_ID = local.peer_id
195+
DATABASE_PRIMARY_RPC_SERVER_PORT = tostring(local.db_primary_rpc_server_port)
196+
DATABASE_SECONDARY_RPC_SERVER_PORT = tostring(local.db_secondary_rpc_server_port)
197+
SMART_CONTRACT_ADDRESS = local.smart_contract_address
198+
}
189199

190-
secrets = merge({
191-
SECRET_KEY = module.secret["ed25519_secret_key"]
192-
SMART_CONTRACT_ENCRYPTION_KEY = module.secret["smart_contract_encryption_key"]
193-
RPC_PROVIDER_URL = module.secret["rpc_provider_url"]
194-
},
195-
# configure pk only for the primary node
196-
count.index != 0 ? {} : {
197-
SMART_CONTRACT_SIGNER_PRIVATE_KEY = module.secret["ecdsa_private_key"]
198-
})
200+
secrets = merge({
201+
SECRET_KEY = module.secret["ed25519_secret_key"]
202+
SMART_CONTRACT_ENCRYPTION_KEY = module.secret["smart_contract_encryption_key"]
203+
RPC_PROVIDER_URL = module.secret["rpc_provider_url"]
204+
},
205+
# configure pk only for the primary node
206+
count.index != 0 ? {} : {
207+
SMART_CONTRACT_SIGNER_PRIVATE_KEY = module.secret["ecdsa_private_key"]
208+
})
209+
}]
199210
})
200211
}
201212

202213
locals {
203214
prometheus_port = 3000
215+
prom2parquet_port = 4000
204216
prometheus_domain_name = try("prometheus.${local.region_prefix}.${var.config.route53_zone.name}", null)
205217
}
206218

@@ -219,6 +231,10 @@ module "prometheus_config" {
219231
targets = ["${module.node[0].private_ip}:${local.node_metrics_server_port}"]
220232
}]
221233
}]
234+
remote_write = local.prometheus_s3_export ? [{
235+
url = "http://localhost:${local.prom2parquet_port}/receive"
236+
remote_timeout = "30s"
237+
}] : []
222238
})
223239
}
224240

@@ -250,27 +266,56 @@ module "prometheus" {
250266

251267
public_ip = false
252268

253-
ports = [
254-
{ port = local.prometheus_port, protocol = "tcp", internal = true },
255-
]
256-
257-
environment = {}
258-
secrets = {
259-
CONFIG = module.prometheus_config[0]
260-
WEB_CONFIG = module.prometheus_web_config[0]
261-
}
262-
263-
entry_point = ["/bin/sh", "-c"]
264-
command = [<<-CMD
265-
printenv CONFIG > /tmp/prometheus.yml && \
266-
printenv WEB_CONFIG > /tmp/web.yml && \
267-
exec /bin/prometheus \
268-
--config.file=/tmp/prometheus.yml \
269-
--web.config.file=/tmp/web.yml \
270-
--web.listen-address=:${local.prometheus_port} \
271-
--storage.tsdb.path=/data
272-
CMD
273-
]
269+
containers = concat(
270+
[{
271+
name = "prometheus"
272+
image = var.config.prometheus.image
273+
ports = [
274+
{ port = local.prometheus_port, protocol = "tcp", internal = true },
275+
]
276+
277+
environment = {}
278+
secrets = {
279+
CONFIG = module.prometheus_config[0]
280+
WEB_CONFIG = module.prometheus_web_config[0]
281+
}
282+
283+
entry_point = ["/bin/sh", "-c"]
284+
command = [<<-CMD
285+
printenv CONFIG > /tmp/prometheus.yml && \
286+
printenv WEB_CONFIG > /tmp/web.yml && \
287+
exec /bin/prometheus \
288+
--config.file=/tmp/prometheus.yml \
289+
--web.config.file=/tmp/web.yml \
290+
--web.listen-address=:${local.prometheus_port} \
291+
--storage.tsdb.path=/data
292+
CMD
293+
]
294+
}],
295+
local.prometheus_s3_export ? [{
296+
name = "prom2parquet"
297+
image = var.config.prometheus.prom2parquet_image
298+
essential = false
299+
ports = [
300+
{ port = local.prom2parquet_port, protocol = "tcp", internal = true },
301+
]
302+
303+
environment = {
304+
AWS_REGION = local.region
305+
}
306+
secrets = {}
307+
308+
entry_point = ["/prom2parquet"]
309+
command = [
310+
"--backend", "s3",
311+
"--backend-root", "${var.config.prometheus.s3_bucket}",
312+
"--prefix", "${var.config.prometheus.s3_metrics_prefix}/${local.region}",
313+
"--server-port", tostring(local.prom2parquet_port),
314+
]
315+
}] : []
316+
)
317+
318+
s3_buckets = var.config.prometheus.s3_bucket == null ? [] : [var.config.prometheus.s3_bucket]
274319
})
275320
}
276321

@@ -318,27 +363,30 @@ module "grafana" {
318363

319364
public_ip = false
320365

321-
ports = [
322-
{ port = local.grafana_port, protocol = "tcp", internal = true },
323-
]
366+
containers = [{
367+
image = var.config.grafana.image
368+
ports = [
369+
{ port = local.grafana_port, protocol = "tcp", internal = true },
370+
]
324371

325-
environment = {
326-
GF_SERVER_HTTP_PORT = tostring(local.grafana_port)
327-
GF_PATHS_DATA = "/data"
328-
GF_SECURITY_ADMIN_USER = "admin"
329-
}
372+
environment = {
373+
GF_SERVER_HTTP_PORT = tostring(local.grafana_port)
374+
GF_PATHS_DATA = "/data"
375+
GF_SECURITY_ADMIN_USER = "admin"
376+
}
330377

331-
secrets = {
332-
GF_SECURITY_ADMIN_PASSWORD = module.secret["grafana_admin_password"]
333-
PROMETHEUS_DATASOURCE_CONFIG = module.grafana_prometheus_datasource_config[0]
334-
}
378+
secrets = {
379+
GF_SECURITY_ADMIN_PASSWORD = module.secret["grafana_admin_password"]
380+
PROMETHEUS_DATASOURCE_CONFIG = module.grafana_prometheus_datasource_config[0]
381+
}
335382

336-
entry_point = ["/bin/sh", "-c"]
337-
command = [<<-CMD
338-
printenv PROMETHEUS_DATASOURCE_CONFIG > /etc/grafana/provisioning/datasources/prometheus.yaml && \
339-
exec /run.sh
340-
CMD
341-
]
383+
entry_point = ["/bin/sh", "-c"]
384+
command = [<<-CMD
385+
printenv PROMETHEUS_DATASOURCE_CONFIG > /etc/grafana/provisioning/datasources/prometheus.yaml && \
386+
exec /run.sh
387+
CMD
388+
]
389+
}]
342390
})
343391
}
344392

0 commit comments

Comments
 (0)