diff --git a/reports/analytics/analytics_dashboard.qmd b/reports/analytics/analytics_dashboard.qmd index 80226cb..c6b2cb3 100644 --- a/reports/analytics/analytics_dashboard.qmd +++ b/reports/analytics/analytics_dashboard.qmd @@ -4,6 +4,18 @@ execute: echo: false warning: false message: false +params: + # Dates must be inserted as YYYY-MM-DD. + # Lists of IP's or excluded dates are prefixed by '-'. + # All params are optional and will default + # to querying the first day of data to the + # current day. + start_date: + excluded_dates: + end_date: + # If false, this will still return cached data for today + refresh_today: true + ip_addresses_to_filter: format: html: embed-resources: true @@ -28,8 +40,47 @@ library(urltools) library(uaparserjs) ``` +```{r convert_params} +tz_local <- "America/Chicago" +today_local <- today(tzone = tz_local) -```{r} +ip_addresses <- c( + if (!is.null(params$ip_addresses_to_filter)) { + params$ip_addresses_to_filter + } else { + character(0) + } +) + +excluded_dates <- c( + if (!is.null(params$excluded_dates)) { + as.Date(params$excluded_dates) + } else { + as.Date(character(0)) + } +) + +refresh_today <- if (!is.null(params$refresh_today)) { + params$refresh_today +} else { + TRUE +} + +# Set start day to the first day of data unless otherwise specified +start_date <- as.Date(ifelse( + is.null(params$start_date), + "2025-09-04", + params$start_date +)) + +# Set end date to today unless otherwise specified +end_date <- as.Date( + if (is.null(params$end_date)) today_local else params$end_date +) +``` + + +```{r download_assessment_card_data} conn <- dbConnect(noctua::athena(), rstudio_conn_tab = FALSE) cache_dir <- "data/assessment_card" @@ -52,10 +103,7 @@ if (!file.exists(cache_file)) { } ``` -```{r} -tz_local <- "America/Chicago" -baseline_date <- as.Date("2025-09-04") - +```{r download_logs} log_group_name <- "/aws-cloudfront/pinval-prod" log_stream_name <- "CloudFront_E3AIN3GTEPCCYR" @@ -125,9 +173,6 @@ fetch_events_for_day <- function(date_local) { ) } -# Today in local time -today_local <- today(tzone = tz_local) - # Look for already-cached files like logs_YYYY-MM-DD.parquet cached_files <- dir_ls(out_dir, regexp = "logs_\\d{4}-\\d{2}-\\d{2}\\.parquet$", @@ -141,14 +186,20 @@ cached_dates <- str_match( cached_dates <- as.Date(na.omit(cached_dates)) +# Remove today from cached dates if refresh_today is not FALSE +if (refresh_today != FALSE) { + cached_dates <- setdiff(cached_dates, today_local) +} + # Build the full sequence of dates -all_dates <- seq(baseline_date, today_local, by = "day") +all_dates <- seq(start_date, end_date, by = "day") -dates_to_fetch <- c( - # Always call today's date - setdiff(all_dates[all_dates != today_local], cached_dates), - today_local -) %>% sort() +# Remove days which should be excluded via the params and +# cached files. +dates_to_fetch <- setdiff( + setdiff(all_dates, cached_dates), + excluded_dates +) message( "Dates to fetch (", length(dates_to_fetch), "): ", @@ -159,11 +210,6 @@ walk(dates_to_fetch, function(d) { out_file <- fs::path(out_dir, paste0("logs_", format(d), ".parquet")) message("Target file: ", fs::path_abs(out_file)) - if (fs::file_exists(out_file) && d != today_local) { - message("Skipping ", format(d), " (already cached).") - return(invisible(NULL)) - } - message("Fetching ", format(d), " ...") df <- fetch_events_for_day(d) @@ -173,17 +219,30 @@ walk(dates_to_fetch, function(d) { message("Verified exists: ", as.character(fs::file_size(out_file)), " bytes") }) - +# List all log files logs <- fs::dir_ls( out_dir, regexp = "logs_\\d{4}-\\d{2}-\\d{2}\\.parquet$", type = "file" -) %>% - purrr::map_dfr(arrow::read_parquet) +) + +# Extract date from filenames +log_dates <- stringr::str_extract(basename(logs), "\\d{4}-\\d{2}-\\d{2}") %>% + as.Date() + +# Filter imports based on params +logs_to_import <- tibble(file = logs, date = log_dates) %>% + filter( + date <= end_date, + date >= start_date, + !(date %in% excluded_dates) + ) + +logs <- purrr::map_dfr(logs_to_import$file, arrow::read_parquet) ``` -```{r} +```{r parse_data} fields <- c( "date", "time", "sc-bytes", "c-ip", "cs-method", "cs(Host)", "cs-uri-stem", "sc-status", "cs(Referer)", "cs(User-Agent)", @@ -205,6 +264,9 @@ data <- logs %>% })) %>% select(-any_of(fields)) %>% tidyr::unnest_wider(parsed, names_repair = "minimal") %>% + filter( + !(`c-ip` %in% params$ip_addresses_to_filter) + ) %>% mutate( pin = coalesce( str_extract(`cs-uri-stem`, "(?<=/)\\d{14}"), @@ -251,7 +313,7 @@ NA values are for PINs which do not have a matching assessment_card record. ### Overall -```{r} +```{r class_overall} # We want to keep multi-cards just for the class related plots data_class <- data %>% filter(uri_indicator == "Query") %>% @@ -291,7 +353,7 @@ data_class %>% ### Count Over Time -```{r} +```{r class_time} data_class %>% group_by(date, char_class) %>% summarise(request_count = n(), .groups = "drop") %>% @@ -322,7 +384,7 @@ data_class %>% ### % Over Time -```{r} +```{r class_percent_time} data_class %>% mutate(date = as.Date(datetime)) %>% group_by(date, char_class) %>% @@ -365,7 +427,7 @@ Extract year refers to the year which the user queried the data for. ### Overall -```{r} +```{r count_by_year} # This removes the duplicate cards from the previous join. data <- data %>% distinct(message, .keep_all = TRUE) %>% @@ -393,7 +455,7 @@ data %>% ### Total Over Time -```{r} +```{r count_over_time} data %>% filter( uri_indicator == "Query" @@ -427,7 +489,7 @@ data %>% ### % Over Time -```{r} +```{r percent_over_time} data %>% filter( uri_indicator == "Query" @@ -471,7 +533,7 @@ data %>% ### Overall -```{r} +```{r eligibility_overall} data %>% filter(uri_indicator == "Query") %>% mutate( @@ -502,7 +564,7 @@ data %>% This allows for duplicate reasons if the same query is deemed ineligible. -```{r} +```{r eligibility_reasons} ineligbility_data <- data %>% filter( uri_indicator == "Query", @@ -541,7 +603,7 @@ ineligbility_data %>% ### Count Over Time -```{r} +```{r eligibility_time} ineligbility_data %>% count(reason_clean, date, name = "request_count") %>% plot_ly( @@ -571,7 +633,7 @@ ineligbility_data %>% ### % Over Time -```{r} +```{r eligibility_percent_time} ineligbility_data %>% count(date, reason_report_ineligible, name = "request_count") %>% complete(date, reason_report_ineligible, fill = list(request_count = 0)) %>% @@ -604,7 +666,7 @@ ineligbility_data %>% --- -```{r} +```{r prepare_township_year} df_counts <- data %>% filter(uri_indicator == "Query") %>% mutate( @@ -649,7 +711,7 @@ years <- sort(unique(df_counts$extract_year)) ### Overall -```{r} +```{r township_overall} data %>% filter(uri_indicator == "Query") %>% transmute( @@ -678,13 +740,13 @@ data %>% ### 2024 -```{r} +```{r 2024} plot_township_year(df_counts, "2024") ``` ### 2025 -```{r} +```{r 2025} plot_township_year(df_counts, "2025") ``` ::: @@ -697,7 +759,7 @@ This map filters out any queries for PINs that do not have coordinates in the underlying data, which is always true for non-tri PINs. As such, any queries for non-tri PINs (e.g. north suburbs PINs in 2024) will not appear on this map. -```{r} +```{r map_distinct_pins} df <- data %>% filter(uri_indicator == "Query") %>% distinct(pin, .keep_all = TRUE) %>% @@ -738,7 +800,7 @@ leaflet(df) %>% ### Excluding Static File Requests -```{r} +```{r http_status_excluding_static} data %>% mutate( date = as.Date(datetime), @@ -771,7 +833,7 @@ data %>% ### Including Static File Requests -```{r} +```{r http_status_including_static} data %>% mutate( date = as.Date(datetime), @@ -803,7 +865,7 @@ data %>% ### Recent Errors (non-2xx/3xx status codes) -```{r} +```{r recent_errors} data %>% # Keep only errors (not 2xx or 3xx) filter(!is.na(`sc-status`), !substr(`sc-status`, 1, 1) %in% c("2", "3")) %>% @@ -836,7 +898,7 @@ data %>% ### Excluding Static File Requests -```{r} +```{r common_browsers_excluding_static} normalize_browser <- function(x) { x <- str_to_lower(str_squish(x)) case_when( @@ -878,7 +940,7 @@ data %>% ### Including Static File Requests -```{r} +```{r common_browsers_including_static} data %>% mutate(parsed = ua_parse(`cs(User-Agent)`)) %>% unnest_wider(parsed) %>% @@ -909,7 +971,7 @@ data %>% ### Excluding Static File Requests -```{r} +```{r common_referer_excluding_static} data %>% filter(uri_indicator == "Query") %>% mutate( @@ -939,7 +1001,7 @@ data %>% ### Including Static File Requests -```{r} +```{r common_referer_including_static} data %>% mutate( referer_raw = `cs(Referer)`, @@ -977,7 +1039,7 @@ data %>% ### Excluding Static File Requests -```{r} +```{r cache_hit_excluding_static} data %>% filter(uri_indicator == "Query") %>% mutate( @@ -1015,7 +1077,7 @@ data %>% ### Including Static File Requests -```{r} +```{r cache_hit_including_static} data %>% mutate( result_type = as.character(`x-edge-response-result-type`), @@ -1058,7 +1120,7 @@ data %>% ### Time to First byte -```{r} +```{r time_to_first_byte} data %>% filter(uri_indicator != "Other") %>% mutate( @@ -1080,7 +1142,7 @@ data %>% ### Time Taken -```{r} +```{r time_taken} data %>% filter(uri_indicator != "Other") %>% mutate( diff --git a/reports/analytics/renv.lock b/reports/analytics/renv.lock index 7cf89ac..294c24d 100644 --- a/reports/analytics/renv.lock +++ b/reports/analytics/renv.lock @@ -103,7 +103,7 @@ }, "arrow": { "Package": "arrow", - "Version": "21.0.0.1", + "Version": "22.0.0", "Source": "Repository", "Repository": "CRAN", "Requirements": [ @@ -121,7 +121,7 @@ "utils", "vctrs" ], - "Hash": "ef54a31808517d2197e83dd57ca391e5" + "Hash": "833507e330290fc24b2d09f1737cab14" }, "askpass": { "Package": "askpass", @@ -1051,6 +1051,16 @@ ], "Hash": "d5811dade9cd3a34a9038a45f1a59bae" }, + "otel": { + "Package": "otel", + "Version": "0.2.0", + "Source": "Repository", + "Repository": "CRAN", + "Requirements": [ + "R" + ], + "Hash": "627d6993db1043703c0b084fa432f21f" + }, "paws": { "Package": "paws", "Version": "0.9.0", @@ -1331,19 +1341,20 @@ }, "promises": { "Package": "promises", - "Version": "1.3.3", + "Version": "1.4.0", "Source": "Repository", "Repository": "CRAN", "Requirements": [ + "R", "R6", - "Rcpp", "fastmap", "later", + "lifecycle", "magrittr", - "rlang", - "stats" + "otel", + "rlang" ], - "Hash": "70157a4a73963da5779f9c67f4d31fd1" + "Hash": "e11326800dde656c4813afff144d8eb9" }, "proxy": { "Package": "proxy", @@ -2015,7 +2026,7 @@ }, "xfun": { "Package": "xfun", - "Version": "0.53", + "Version": "0.54", "Source": "Repository", "Repository": "CRAN", "Requirements": [ @@ -2024,11 +2035,11 @@ "stats", "tools" ], - "Hash": "1befe97a0d56060b3964e3986d408df9" + "Hash": "d23c8a31e000d9865a8a6572f61da9ee" }, "xml2": { "Package": "xml2", - "Version": "1.4.0", + "Version": "1.4.1", "Source": "Repository", "Repository": "CRAN", "Requirements": [ @@ -2037,7 +2048,7 @@ "methods", "rlang" ], - "Hash": "3dfd9ffbe221af775cb37f95e7e893ec" + "Hash": "00841b97c923064fdffb4f20db816c98" }, "yaml": { "Package": "yaml",