Skip to content

Commit d88307a

Browse files
nikhilsinhaparseablenikhilsinhaparseable
committed
add scanned metrics by date
1 parent 0e46ecc commit d88307a

File tree

10 files changed

+354
-86
lines changed

10 files changed

+354
-86
lines changed

‎src/metadata.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,10 +62,10 @@ pub fn update_stats(
6262
.add(size as i64);
6363
TOTAL_EVENTS_INGESTED_DATE
6464
.with_label_values(&[origin, &parsed_date])
65-
.inc_by(num_rows as u64);
65+
.add(num_rows as i64);
6666
TOTAL_EVENTS_INGESTED_SIZE_DATE
6767
.with_label_values(&[origin, &parsed_date])
68-
.inc_by(size);
68+
.add(size as i64);
6969
}
7070

7171
/// In order to support backward compatability with streams created before v1.6.4,

‎src/metrics/mod.rs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -160,8 +160,8 @@ pub static EVENTS_STORAGE_SIZE_DATE: Lazy<IntCounterVec> = Lazy::new(|| {
160160
.expect("metric can be created")
161161
});
162162

163-
pub static TOTAL_EVENTS_INGESTED_DATE: Lazy<IntCounterVec> = Lazy::new(|| {
164-
IntCounterVec::new(
163+
pub static TOTAL_EVENTS_INGESTED_DATE: Lazy<IntGaugeVec> = Lazy::new(|| {
164+
IntGaugeVec::new(
165165
Opts::new(
166166
"total_events_ingested_date",
167167
"total events ingested on a particular date",
@@ -172,8 +172,8 @@ pub static TOTAL_EVENTS_INGESTED_DATE: Lazy<IntCounterVec> = Lazy::new(|| {
172172
.expect("metric can be created")
173173
});
174174

175-
pub static TOTAL_EVENTS_INGESTED_SIZE_DATE: Lazy<IntCounterVec> = Lazy::new(|| {
176-
IntCounterVec::new(
175+
pub static TOTAL_EVENTS_INGESTED_SIZE_DATE: Lazy<IntGaugeVec> = Lazy::new(|| {
176+
IntGaugeVec::new(
177177
Opts::new(
178178
"total_events_ingested_size_date",
179179
"Total events ingested size in bytes on a particular date",
@@ -184,8 +184,8 @@ pub static TOTAL_EVENTS_INGESTED_SIZE_DATE: Lazy<IntCounterVec> = Lazy::new(|| {
184184
.expect("metric can be created")
185185
});
186186

187-
pub static TOTAL_EVENTS_STORAGE_SIZE_DATE: Lazy<IntCounterVec> = Lazy::new(|| {
188-
IntCounterVec::new(
187+
pub static TOTAL_EVENTS_STORAGE_SIZE_DATE: Lazy<IntGaugeVec> = Lazy::new(|| {
188+
IntGaugeVec::new(
189189
Opts::new(
190190
"total_events_storage_size_date",
191191
"Total events storage size in bytes on a particular date",

‎src/metrics/storage.rs‎

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,12 +44,24 @@ pub static STORAGE_FILES_SCANNED: Lazy<CounterVec> = Lazy::new(|| {
4444
.expect("metric can be created")
4545
});
4646

47+
pub static STORAGE_FILES_SCANNED_DATE: Lazy<CounterVec> = Lazy::new(|| {
48+
CounterVec::new(
49+
Opts::new(
50+
"storage_files_scanned_date_total",
51+
"Total number of files scanned in storage operations by date",
52+
)
53+
.namespace(METRICS_NAMESPACE),
54+
&["provider", "operation", "date"],
55+
)
56+
.expect("metric can be created")
57+
});
58+
4759
pub trait StorageMetrics {
4860
fn register_metrics(&self, handler: &PrometheusMetrics);
4961
}
5062

5163
pub mod localfs {
52-
use crate::storage::FSConfig;
64+
use crate::{metrics::storage::STORAGE_FILES_SCANNED_DATE, storage::FSConfig};
5365

5466
use super::{STORAGE_FILES_SCANNED, STORAGE_REQUEST_RESPONSE_TIME, StorageMetrics};
5567

@@ -63,12 +75,16 @@ pub mod localfs {
6375
.registry
6476
.register(Box::new(STORAGE_FILES_SCANNED.clone()))
6577
.expect("metric can be registered");
78+
handler
79+
.registry
80+
.register(Box::new(STORAGE_FILES_SCANNED_DATE.clone()))
81+
.expect("metric can be registered");
6682
}
6783
}
6884
}
6985

7086
pub mod s3 {
71-
use crate::storage::S3Config;
87+
use crate::{metrics::storage::STORAGE_FILES_SCANNED_DATE, storage::S3Config};
7288

7389
use super::{STORAGE_FILES_SCANNED, STORAGE_REQUEST_RESPONSE_TIME, StorageMetrics};
7490

@@ -82,12 +98,16 @@ pub mod s3 {
8298
.registry
8399
.register(Box::new(STORAGE_FILES_SCANNED.clone()))
84100
.expect("metric can be registered");
101+
handler
102+
.registry
103+
.register(Box::new(STORAGE_FILES_SCANNED_DATE.clone()))
104+
.expect("metric can be registered");
85105
}
86106
}
87107
}
88108

89109
pub mod azureblob {
90-
use crate::storage::AzureBlobConfig;
110+
use crate::{metrics::storage::STORAGE_FILES_SCANNED_DATE, storage::AzureBlobConfig};
91111

92112
use super::{STORAGE_FILES_SCANNED, STORAGE_REQUEST_RESPONSE_TIME, StorageMetrics};
93113

@@ -101,12 +121,16 @@ pub mod azureblob {
101121
.registry
102122
.register(Box::new(STORAGE_FILES_SCANNED.clone()))
103123
.expect("metric can be registered");
124+
handler
125+
.registry
126+
.register(Box::new(STORAGE_FILES_SCANNED_DATE.clone()))
127+
.expect("metric can be registered");
104128
}
105129
}
106130
}
107131

108132
pub mod gcs {
109-
use crate::storage::GcsConfig;
133+
use crate::{metrics::storage::STORAGE_FILES_SCANNED_DATE, storage::GcsConfig};
110134

111135
use super::{STORAGE_FILES_SCANNED, STORAGE_REQUEST_RESPONSE_TIME, StorageMetrics};
112136

@@ -120,6 +144,10 @@ pub mod gcs {
120144
.registry
121145
.register(Box::new(STORAGE_FILES_SCANNED.clone()))
122146
.expect("metric can be registered");
147+
handler
148+
.registry
149+
.register(Box::new(STORAGE_FILES_SCANNED_DATE.clone()))
150+
.expect("metric can be registered");
123151
}
124152
}
125153
}

‎src/query/stream_schema_provider.rs‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,10 @@ use crate::{
5757
},
5858
event::DEFAULT_TIMESTAMP_KEY,
5959
hottier::HotTierManager,
60-
metrics::{QUERY_CACHE_HIT, storage::STORAGE_FILES_SCANNED},
60+
metrics::{
61+
QUERY_CACHE_HIT,
62+
storage::{STORAGE_FILES_SCANNED, STORAGE_FILES_SCANNED_DATE},
63+
},
6164
option::Mode,
6265
parseable::{PARSEABLE, STREAM_EXISTS},
6366
storage::{ObjectStorage, ObjectStorageError, ObjectStoreFormat},
@@ -586,6 +589,13 @@ impl TableProvider for StandardTableProvider {
586589
STORAGE_FILES_SCANNED
587590
.with_label_values(&[PARSEABLE.storage().name(), "GET"])
588591
.inc_by(parquet_files_to_scan as f64);
592+
STORAGE_FILES_SCANNED_DATE
593+
.with_label_values(&[
594+
PARSEABLE.storage().name(),
595+
"GET",
596+
&Utc::now().date_naive().to_string(),
597+
])
598+
.inc_by(parquet_files_to_scan as f64);
589599

590600
let (partitioned_files, statistics) = self.partitioned_files(manifest_files);
591601
self.create_parquet_physical_plan(

‎src/stats.rs‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -137,13 +137,19 @@ pub async fn update_deleted_stats(
137137
"parquet",
138138
&manifest_date,
139139
]);
140-
let _ = TOTAL_EVENTS_INGESTED_DATE.remove_label_values(&["json", &manifest_date]);
141-
let _ = TOTAL_EVENTS_INGESTED_SIZE_DATE.remove_label_values(&["json", &manifest_date]);
142-
let _ =
143-
TOTAL_EVENTS_STORAGE_SIZE_DATE.remove_label_values(&["parquet", &manifest_date]);
140+
144141
num_row += manifest.events_ingested as i64;
145142
ingestion_size += manifest.ingestion_size as i64;
146143
storage_size += manifest.storage_size as i64;
144+
TOTAL_EVENTS_INGESTED_DATE
145+
.with_label_values(&["json", &manifest_date])
146+
.sub(manifest.events_ingested as i64);
147+
TOTAL_EVENTS_INGESTED_SIZE_DATE
148+
.with_label_values(&["json", &manifest_date])
149+
.sub(manifest.ingestion_size as i64);
150+
TOTAL_EVENTS_STORAGE_SIZE_DATE
151+
.with_label_values(&["parquet", &manifest_date])
152+
.sub(manifest.storage_size as i64);
147153
}
148154
}
149155
EVENTS_DELETED

0 commit comments

Comments
 (0)