diff --git a/.gitmodules b/.gitmodules index fb7f2dfd7..0d68b33bb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "jpo-utils"] path = jpo-utils - url = https://github.com/usdot-jpo-ode/jpo-utils + url = https://github.com/CDOT-CV/jpo-utils branch = master diff --git a/.vscode/launch.json b/.vscode/launch.json index 8ff266510..25da7cb11 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -35,7 +35,24 @@ "stopOnEntry": false, "program": "${workspaceRoot}/services/api/src/main.py", "env": { - "FLASK_APP": "${workspaceRoot}/services/api/src/main.py" + "PYTHONPATH": "${workspaceRoot}/services", + "FLASK_APP": "${workspaceRoot}/services/api/src/main.py", + "FLASK_RUN_PORT": "8081", + "PG_DB_HOST": "localhost:5432", + "PG_DB_USER": "postgres", + "PG_DB_PASS": "postgres", + "PG_DB_NAME": "postgres", + "MONGO_DB_URI": "mongodb://ode:replace-me@localhost:27017/?directConnection=true&authSource=admin", + "MONGO_DB_NAME": "CV", + "MONGO_PROCESSED_BSM_COLLECTION_NAME": "processed_bsm", + "MONGO_PROCESSED_PSM_COLLECTION_NAME": "processed_psm", + "ENABLE_WZDX_FEATURES": "false", + "ENABLE_MOOVE_AI_FEATURES": "false", + "CORS_DOMAIN": "*", + "KEYCLOAK_ENDPOINT": "http://localhost:8084", + "KEYCLOAK_REALM": "cvmanager", + "KEYCLOAK_API_CLIENT_ID": "cvmanager-api", + "KEYCLOAK_API_CLIENT_SECRET_KEY": "keycloak-secret-key" }, "args": ["run"], "envFile": "${workspaceRoot}/services/api/.env", @@ -48,13 +65,13 @@ "mainClass": "us.dot.its.jpo.ode.api.ConflictApiApplication", "projectName": "intersection-api", "envFile": "${workspaceFolder}/.env", - "preLaunchTask": "run-docker-compose-intersection-no-api" + "preLaunchTask": "run-intersection-no-api" } ], "compounds": [ { "name": "Debug Solution", - "configurations": ["Python: Flask", "Launch web app with Intersection Data"] + "configurations": ["Python: Flask", "Launch web app with Intersection API"] } ] } diff --git a/.vscode/settings.json b/.vscode/settings.json index 524d22b21..78d90f30c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -88,6 +88,7 @@ "OIDC", "OIDCID", "pgdb", + "pgquery", "PGSQL", "postgis", "pythjon", diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 4096226de..561fb8e3b 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -15,44 +15,45 @@ "command": "${command:python.interpreterPath} -m pytest -v --cov-report xml:cov.xml --cov ." }, { - "label": "run-intersection", + "label": "run-keycloak-and-postgres", "type": "docker-compose", "dockerCompose": { "up": { "detached": true, "build": true, - "profiles": ["cvmanager_postgres", "cvmanager_keycloak", "intersection"] + "profiles": ["cvmanager_postgres", "cvmanager_keycloak"] }, "files": ["${workspaceFolder}/docker-compose.yml"], "envFile": "${workspaceFolder}/.env" - }, - "dependsOn": ["run-keycloak-and-postgres"] + } }, { - "label": "run-docker-compose-intersection-no-api", + "label": "run-intersection", "type": "docker-compose", "dockerCompose": { "up": { "detached": true, "build": true, - "profiles": ["intersection_no_api", "cvmanager_api", "mongo_full", "kafka_full", "kafka_connect_standalone"] + "profiles": ["intersection", "mongo_full"] }, - "files": ["${workspaceFolder}/docker-compose.yml", "${workspaceFolder}/docker-compose-intersection.yml"], + "files": ["${workspaceFolder}/docker-compose.yml"], "envFile": "${workspaceFolder}/.env" - } + }, + "dependsOn": ["run-keycloak-and-postgres"] }, { - "label": "run-keycloak-postgres-mongodb-kafka", + "label": "run-intersection-no-api", "type": "docker-compose", "dockerCompose": { "up": { "detached": true, "build": true, - "services": ["cvmanager_keycloak", "cvmanager_postgres"] + "profiles": ["mongo_full"] }, "files": ["${workspaceFolder}/docker-compose.yml"], "envFile": "${workspaceFolder}/.env" - } + }, + "dependsOn": ["run-keycloak-and-postgres"] }, { "label": "run-full-conflictmonitor", @@ -61,24 +62,12 @@ "up": { "detached": true, "build": true, - "services": [ - "cvmanager_postgres", - "cvmanager_keycloak", - "kafka", - "kafka_init", - "ode", - "geojsonconverter", - "conflictmonitor", - "deduplicator", - "conflictvisualizer_api", - "mongodb_container", - "connect" - ] + "services": ["kafka_full", "kafka_connect_standalone", "conflictmonitor"] }, - "files": ["${workspaceFolder}/docker-compose-full-cm.yml"], + "files": ["${workspaceFolder}/docker-compose.yml"], "envFile": "${workspaceFolder}/.env" }, - "dependsOn": ["run-keycloak-and-postgres"] + "dependsOn": ["run-intersection"] } ] } diff --git a/README.md b/README.md index 6e881e0ab..7ed8f723f 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,6 @@ The JPO Connected Vehicle Manager is a web-based application that helps an organization manage their deployed CV devices (Roadside Units and Onboard Units) through an interactive, graphical user interface using Mapbox. -**GUI:** ReactJS with Redux Toolkit and Mapbox GL - -**API:** Python 3.12.2 - **Features:** - Visualize devices on a Mapbox map @@ -28,6 +24,56 @@ The JPO Connected Vehicle Manager is a web-based application that helps an organ To provide feedback, we recommend that you create an "issue" in this repository (). You will need a GitHub account to create an issue. If you don’t have an account, a dialog will be presented to you to create one at no cost. +## Quick Start + +This section is brief - for more detailed instructions, please see [Getting Started](#getting-started) + +### Requirements + +- Docker and Docker Compose installed on your machine + +### Steps + +1. Copy the `sample.env` file to a new file named `.env` in the root directory of the project. +2. Edit the `.env` file to set the required environment variables. At a minimum, you will need to set the following variables: + - `DOCKER_HOST_IP`: The IP address of your Docker host. This can be found through linux/wsl through the command "ifconfig", or "localhost" if using Docker Desktop on Windows or Linux (not mac). + - `MAPBOX_TOKEN`: Any valid mapbox token. Please see [Creating a Mapbox Token](#creating-a-mapbox-token) for instructions on how to create and account/generate a new token + - `MAVEN_GITHUB_TOKEN`: A GitHub access token used to access public GitHub Maven packages. See [Github Token](#github-token) section for instructions on generating this token. +3. Initialize the jpo-utils submodule: + +```sh +git submodule update --init --recursive +``` + +4. Run the following command to start the CV Manager: + +```sh +docker compose up -d +``` + +5. Access the CV Manager webapp at [http://localhost:3000](http://localhost:3000) in your web browser. + +``` +Default Username: test@gmail.com +Default Password: tester +``` + +If you have any issues, try the following steps: + +1. Ensure that the following required services are healthy in Docker (docker ps): + i. cvmanager_postgres + ii. cvmanager_keycloak + iii. cvmanager_api + iv. cvmanager_webapp +2. Bring down the system and re-build all containers + +```sh +docker compose down -v +docker compose up --build -d +``` + +For more details on running the CV-Manager through Docker, see the [Getting Started](#getting-started) section below. + ## Release Notes The current version and release history of the JPO CV Manager: [Release Notes](docs/Release_notes.md) @@ -40,10 +86,31 @@ The JPO CV Manager was originally developed for the Google Cloud Platform and a ### CV Manager Webapp +ReactJS with Redux Toolkit and Mapbox GL + - Supports OAuth2.0 through Keycloak for user authentication only. It can be configured for several different Identity Providers, including Google. +#### Creating a Mapbox Token + +To generate a free mapbox access token: + +1. navigate to https://www.mapbox.com/ and select "Get started for free" +2. Enter your name and email, and choose a unique username +3. Enter your payment information. Your account will remain free unless you surpass the free tier limits (the free tier is very extensive, it will cover everything up to a multi-user large scale deployment) +4. Confirm your email address +5. Enter your billing address +6. After logging in, select the "Tokens" tab under the Admin section +7. Press create a new token (it is easier to manage and re-create new tokens than the default public token) + i. Enter a recognizable token name + ii. No scopes are required, as this token will only be used for tile loading + iii. Under Token Restrictions, enter the URL paths the CV-Manager will be hosted on. For local development, this is http://localhost:3000 and http://${DOCKER_HOST_IP}:3000. This is incredibly important. When the CV-Manager is deployed, the mapbox token can be extracted quite easily. The only way to protect the use of this token (and not incur additional access costs) is to restrict the allowed domains + iv. Select "Create Token" +8. Copy the token value and paste it into the .env under MAPBOX_TOKEN= + ### CV Manager API +Python 3.12.2 + - PostgreSQL database is required. Run the [table creation script to create a to-spec database](resources/sql_scripts). - Follow along with the README to ensure your data is properly populated before running the CV Manager. - GCP BigQuery is required to support J2735 message counts and BSM data. Message counts will be migrated to PostgreSQL eventually, however it is not recommended to store full J2735 messages in a PostgreSQL database. A noSQL database or a database that is specialized for storing big data is recommended. Support for MongoDB is planned to be implemented. @@ -117,13 +184,6 @@ Ease of local development has been a major consideration in the integration of i It should be noted that the `kafka`, `kafka-setup`, `mongo` and `mongo-setup` services are provided by the jpo-utils repository. -**Intersection API Submodules** -The Intersection API uses nested submodules for asn1 encoding and decoding [usdot-jpo-ode/asn1_codec](https://github.com/usdot-jpo-ode/asn1_codec) and kafka management. These submodules need to be initialized and updated before the API can be built and run locally. Run the following command to initialize the submodules: - -```sh -git submodule update --init --recursive -``` - **Running a Simple Local Environment** Build the docker-compose: @@ -207,18 +267,15 @@ The following steps are intended to help get a new user up and running the JPO C git submodule update --init --recursive ``` 3. Create a copy of the sample.env named ".env" and refer to the Environmental variables section below for more information on each variable. - 1. Make sure at least the DOCKER_HOST_IP, KEYCLOAK_ADMIN_PASSWORD, KEYCLOAK_API_CLIENT_SECRET_KEY, and MAPBOX_TOKEN are set for this. - 2. Some of these variables, delineated by sections, pertain to the [jpo-conflictmonitor](https://github.com/usdot-jpo-ode/jpo-conflictmonitor), [jpo-geojsonconverter](https://github.com/usdot-jpo-ode/jpo-geojsonconverter), and [jpo-ode](https://github.com/usdot-jpo-ode/jpo-ode). Please see the documentation provided for these projects when setting these variables. -4. The CV Manager has four components that need to be containerized and deployed: the API, the PostgreSQL database, Keycloak, and the webapp. + 1. Make sure at least the DOCKER_HOST_IP, MAVEN_GITHUB_TOKEN, and MAPBOX_TOKEN are set for this. + 2. For other services or different configuration, please make a copy of the sample-full.env. Some of these variables, delineated by sections, pertain to the [jpo-conflictmonitor](https://github.com/usdot-jpo-ode/jpo-conflictmonitor), [jpo-geojsonconverter](https://github.com/usdot-jpo-ode/jpo-geojsonconverter), and [jpo-ode](https://github.com/usdot-jpo-ode/jpo-ode). Please see the documentation provided for these projects when setting these variables. +4. The CV Manager has four core components that need to be built and run: the API, the PostgreSQL database, Keycloak, and the webapp. Ensure that both of the following profiles are specified in the COMPOSE_PROFILES variable of your .env file: - - If you are looking to deploy the CV Manager locally, you can simply run the docker-compose, make sure to fill out the .env file to ensure it launches properly. Also, edit your host file ([How to edit the host file](<[resources/kubernetes](https://docs.rackspace.com/support/how-to/modify-your-hosts-file/)>)) and add IP address of your docker host to these custom domains (remove the carrot brackets and just put the IP address): + - basic: brings up the API, PostgreSQL, and Keycloak + - webapp: brings up the CV-Manager webapp component + - intersection: Optional, brings up the Intersection API and enables visualization/management of connected intersections - CV Manager hosts: - - cvmanager.local.com - cvmanager.auth.com - -5. Apply the docker compose to start the required components: +5. Use the docker compose to start the required components: ```sh docker compose up -d @@ -230,22 +287,22 @@ The following steps are intended to help get a new user up and running the JPO C docker compose up --build -d ``` -6. Access the website by going to: +6. Access the website by going to http://localhost:3000 ``` - http://cvmanager.local.com Default Username: test@gmail.com Default Password: tester ``` -7. To access keycloak go to: +7. To access keycloak go to http://localhost:8084/ ``` - http://cvmanager.auth.com:8084/ Default Username: admin Default Password: admin ``` + This should automatically redirect you to http://${DOCKER_HOST_IP}:8084/. If it does not, navigate to that URL directly. + - If you are looking to deploy in Kubernetes or on separate VMs, refer to the Kubernetes YAML deployment files to deploy the four components to your cluster. ([Kubernetes YAML](resources/kubernetes)) ### Docker Profiles @@ -257,32 +314,31 @@ In addition to the groups defined in the table below, each service may also be a #### Profiles and Services -| Service | basic | webapp | intersection | intersection_no_api | conflictmonitor | addons | obu_ota | -| ---------------------------------- | ----- | ------ | ------------ | ------------------- | --------------- | ------ | ------- | -| cvmanager_api | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| cvmanager_webapp | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | -| cvmanager_postgres | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| cvmanager_keycloak | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| intersection_api | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | -| conflictmonitor | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | -| ode | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | -| aem | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | -| adm | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | -| geojsonconverter | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | -| deduplicator | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | -| connect | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | -| jpo_count_metric | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | -| rsu_status_check | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | -| jpo_iss_health_check | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | -| firmware_manager_upgrade_scheduler | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | -| firmware_manager_upgrade_runner | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | -| jpo_ota_backend | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| jpo_ota_nginx | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| Service | basic | webapp | intersection | conflictmonitor | addons | obu_ota | +| ---------------------------------- | ----- | ------ | ------------ | --------------- | ------ | ------- | +| cvmanager_api | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | +| cvmanager_webapp | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | +| cvmanager_postgres | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | +| cvmanager_keycloak | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | +| intersection_api | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | +| conflictmonitor | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | +| ode | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | +| aem | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | +| adm | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | +| geojsonconverter | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | +| deduplicator | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | +| connect | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | +| jpo_count_metric | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | +| rsu_status_check | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | +| jpo_iss_health_check | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | +| firmware_manager_upgrade_scheduler | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | +| firmware_manager_upgrade_runner | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | +| jpo_ota_backend | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| jpo_ota_nginx | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ##### Note on JPO-Utils Profiles -While `kafka`, `kafka-setup`, `mongo`, `mongo-setup`, and `kafka-connect` are not included in the table above, they are required for the intersection API to run. These services -are provided by the jpo-utils repository. To enable these services, you must include the `kafka_full`, `mongo_full`, and `kafka_connect_standalone` profiles. +While `kafka`, `kafka-setup`, `mongo`, `mongo-setup`, and `kafka-connect` are not included in the table above, they are required for the intersection API to run. These services are provided by the jpo-utils repository. To enable these services, you must include the `kafka_full`, `mongo_full`, and `kafka_connect_standalone` profiles. ### Debugging @@ -321,67 +377,23 @@ Note that it is recommended to work with the Python API from a [virtual environm pip install -r services/requirements.txt ``` -#### Setting up a virtual environment with VSCode - -See [Visual Studio Code](https://code.visualstudio.com/docs/python/environments) documentation for information on how to set up a virtual environment with VS Code. - -#### Debugging Profile - -A debugging profile has been set up for use with VSCode to allow ease of debugging with this application. To use this profile, simply open the project in VSCode and select the "Debug" tab on the left side of the screen. Then, select the "Debug Solution" profile and click the green play button. This will spin up a postgresql instance as well as the keycloak auth solution within docker containers. Once running, this will also start the debugger and attach it to the running API container. You can then set breakpoints and step through the code as needed. - -For the "Debug Solution" to run properly on Windows 10/11 using WSL, the following must be configured: - -1. In a Powershell or Command Prompt terminal run the command: `ifconfig` and open up your `C:\Windows\System32\drivers\etc\hosts` file - - - Copy the `Ethernet adapter vEthernet (WSL) -> IPv4 Address` value to your hosts `cvmanager.auth.com` entry. - - In the same hosts file, update the `cvmanager.local.com` value to: `127.0.0.1`. - -2. Update your main .env file variables as specified in the root of the cvmanager directory - - - Copy the `Ethernet adapter vEthernet (Default) -> IPv4 Address` value to your hosts `WEBAPP_HOST_IP` variable - -3. Apply the docker compose to start the required components: - -```sh -docker compose up -d -``` - -To run only the critical cvmanager components (no intersection services), use this command: - -```sh -docker compose up -d cvmanager_api cvmanager_webapp cvmanager_postgres cvmanager_keycloak -``` - -4. Access the website by going to: - - ``` - http://cvmanager.local.com - Default Username: test@gmail.com - Default Password: tester - ``` - -5. To access keycloak go to: +### Environment Variables - ``` - http://cvmanager.auth.com:8084/ - Default Username: admin - Default Password: admin - ``` +Required Variables -### Environment Variables +- DOCKER_HOST_IP: Set with the IP address of the eth0 port in your WSL instance. This can be found by installing networking tools in wsl and running the command `ifconfig` +- MAPBOX_TOKEN: A token from Mapbox used to render the map in the Webapp. The free version of Mapbox works great in most cases. +- MAVEN_GITHUB_TOKEN: A GitHub access token used to access public GitHub Maven packages. See [Github Token](#github-token) section for instructions on generating this token. Generic Variables -- DOCKER_HOST_IP: Set with the IP address of the eth0 port in your WSL instance. This can be found by installing networking tools in wsl and running the command `ifconfig` - WEBAPP_HOST_IP: Defaults to DOCKER_HOST_IP value. Only change this if the webapp is being hosted on a separate endpoint. - KC_HOST_IP: Defaults to DOCKER_HOST_IP value. Only change this if the webapp is being hosted on a separate endpoint. Webapp Variables -- MAPBOX_TOKEN: A token from Mapbox used to render the map in the Webapp. The free version of Mapbox works great in most cases. - WEBAPP_DOMAIN: The domain that the webapp will run on. This is required for Keycloak CORS authentication. - API_URI: The endpoint for the CV manager API, must be on a Keycloak Authorized domain. -- COUNT_MESSAGE_TYPES: List of CV message types to query for counts. - VIEWER_MSG_TYPES: List of CV message types to query geospatially. - DOT_NAME: The name of the DOT using the CV Manager. - MAPBOX_INIT_LATITUDE: Initial latitude value to use for MapBox view state. @@ -390,11 +402,10 @@ docker compose up -d cvmanager_api cvmanager_webapp cvmanager_postgres cvmanager API Variables -- COUNTS_MSG_TYPES: Set to a list of message types to include in counts query. Sample format is described in the sample.env. - MONGO_PROCESSED_BSM_COLLECTION_NAME: The collection name in MongoDB for processed BSM messages. - MONGO_PROCESSED_PSM_COLLECTION_NAME: The collection name in MongoDB for processed PSM messages. -- SSM_DB_NAME: The database name for SSM visualization data. -- SRM_DB_NAME: The database name for SRM visualization data. +- MONGO_SSM_COLLECTION_NAME: The database name for SSM visualization data. +- MONGO_SRM_COLLECTION_NAME: The database name for SRM visualization data. - FIRMWARE_MANAGER_ENDPOINT: Endpoint for the firmware manager deployment's API. - CSM_EMAIL_TO_SEND_FROM: Origin email address for the API error developer emails. - CSM_EMAILS_TO_SEND_TO: Destination email addresses for the API error developer emails. @@ -447,7 +458,7 @@ git config --global core.autocrlf false - KEYCLOAK_API_CLIENT_ID: Keycloak API client name. - KEYCLOAK_API_CLIENT_SECRET_KEY: Keycloak API secret for the given client name. - KEYCLOAK_LOGIN_THEME_NAME: Name of the jar file to use as the theme provider in Keycloak. For generating a custom theme reference the [Keycloakify](https://github.com/CDOT-CV/keycloakify-starter) Github -- KC_LOGGING_LEVEL: The level of which the Keycloak instance will log. (ALL, DEBUG, ERROR, FATAL, INFO, OFF, TRACE, and WARN) +- KC_LOG_LEVEL: The level of which the Keycloak instance will log. (ALL, DEBUG, ERROR, FATAL, INFO, OFF, TRACE, and WARN) - GOOGLE_CLIENT_ID: GCP OAuth2.0 client ID for SSO Authentication within keycloak. - GOOGLE_CLIENT_SECRET: GCP OAuth2.0 client secret for SSO Authentication within keycloak. @@ -468,6 +479,30 @@ On Windows, Disable `git core.autocrlf` (One Time Only) git config --global core.autocrlf false ``` +### CV Manager Common Issues + +1. After logging into the webapp, you are presented with a page reading "Login Unsuccessful: Unknown Error Occurred" + Login Unsuccessful: Unknown Error Occurred + This indicates an issue within the cvmanager_api service, see the docker logs for more information. Common issues include: + i. Unable to connect to PostgreSQL server (see postgres logs) + ii. Keycloak authentication error (see keycloak logs) +2. The webapp needs to be re-build after each environment variable change + This is due to the fact that environment variables are injected into the Docker image at _BUILD_ time, not runtime. + +```sh +docker compose up --build -d cvmanager_webapp +``` + +3. The Keycloak Hostname needs to be accessible from docker and a browser + Keycloak needs to be accessible other docker containers (cvmanager_api, intersection_api) as well as the webapp (running in a browser). This means that using "localhost" will not work (not accessible from other docker containers), and using "", the docker container network name, will not work either (not accessible in the browser since it is outside of the docker network). This is why the suggested approach is to set your docker host IP address as your keycloak hostname. Keycloak will redirect any incomming connection to the hostname, therefore you cannot set the hostname to "localhost" and have docker services access it at cvmanager_keycloak:8080. +4. The Keycloak volume must be cleared if crucial parameters are changed + If the keycloak admin user credentials, client id(s), client secret, or webapp endpoint are changed by environment variable, those changes will not be reflected in keycloak unless the volume is cleared and re-built. This can be done by: + +```sh +docker compose down -v +docker compose up -d +``` + ## License Information Licensed under the Apache License, Version 2.0 (the "License"); you may not use this diff --git a/docker-compose-addons.yml b/docker-compose-addons.yml index ef59efd39..432595a3b 100644 --- a/docker-compose-addons.yml +++ b/docker-compose-addons.yml @@ -9,32 +9,31 @@ services: image: count_metric:latest restart: on-failure:3 environment: - ENABLE_EMAILER: ${ENABLE_EMAILER} - DEPLOYMENT_TITLE: ${DEPLOYMENT_TITLE} + ENABLE_EMAILER: ${ENABLE_EMAILER:-} + DEPLOYMENT_TITLE: ${DEPLOYMENT_TITLE:-} SMTP_SERVER_IP: ${SMTP_SERVER_IP} SMTP_USERNAME: ${SMTP_USERNAME} SMTP_PASSWORD: ${SMTP_PASSWORD} SMTP_EMAIL: ${SMTP_EMAIL} - MESSAGE_TYPES: ${COUNT_MESSAGE_TYPES} - PROJECT_ID: ${GCP_PROJECT_ID} + PROJECT_ID: ${GCP_PROJECT_ID:-} - PG_DB_HOST: ${PG_DB_HOST} - PG_DB_NAME: ${PG_DB_NAME} - PG_DB_USER: ${PG_DB_USER} - PG_DB_PASS: ${PG_DB_PASS} + PG_DB_HOST: ${PG_DB_HOST:-cvmanager_postgres:5432} + PG_DB_NAME: ${PG_DB_NAME:-postgres} + PG_DB_USER: ${PG_DB_USER:-postgres} + PG_DB_PASS: ${PG_DB_PASS:-postgres} - DESTINATION_DB: ${COUNT_DESTINATION_DB} + DESTINATION_DB: ${COUNT_DESTINATION_DB:-MONGODB} - MONGO_DB_URI: ${MONGO_DB_URI} - MONGO_DB_NAME: ${CM_DATABASE_NAME} - INPUT_COUNTS_MONGO_COLLECTION_NAME: ${INPUT_COUNTS_MONGO_COLLECTION_NAME} - OUTPUT_COUNTS_MONGO_COLLECTION_NAME: ${OUTPUT_COUNTS_MONGO_COLLECTION_NAME} + MONGO_DB_URI: ${MONGO_DB_URI:-mongodb://${MONGO_READ_WRITE_USER:-ode}:${MONGO_READ_WRITE_PASS:-replace-me}@${DB_HOST_IP:-mongo}:${DB_HOST_PORT:-27017}/?directConnection=true&authSource=admin} + MONGO_DB_NAME: ${CM_DATABASE_NAME:-CV} + INPUT_COUNTS_MONGO_COLLECTION_NAME: ${INPUT_COUNTS_MONGO_COLLECTION_NAME:-} + OUTPUT_COUNTS_MONGO_COLLECTION_NAME: ${OUTPUT_COUNTS_MONGO_COLLECTION_NAME:-} - KAFKA_BIGQUERY_TABLENAME: ${KAFKA_BIGQUERY_TABLENAME} + KAFKA_BIGQUERY_TABLENAME: ${KAFKA_BIGQUERY_TABLENAME:-} - LOGGING_LEVEL: ${COUNTS_LOGGING_LEVEL} + LOGGING_LEVEL: ${COUNTS_LOGGING_LEVEL:-} logging: options: max-size: '10m' @@ -50,24 +49,24 @@ services: image: rsu_status_check:latest restart: on-failure:3 environment: - RSU_PING: ${RSU_PING} - ZABBIX: ${ZABBIX} - RSU_MSGFWD_FETCH: ${RSU_MSGFWD_FETCH} - RSU_SECURITY_FETCH: ${RSU_SECURITY_FETCH} - RSU_HEALTH_FETCH: ${RSU_HEALTH_FETCH} + RSU_PING: ${RSU_PING:-True} + ZABBIX: ${ZABBIX:-False} + RSU_MSGFWD_FETCH: ${RSU_MSGFWD_FETCH:-True} + RSU_SECURITY_FETCH: ${RSU_SECURITY_FETCH:-True} + RSU_HEALTH_FETCH: ${RSU_HEALTH_FETCH:-True} - ZABBIX_ENDPOINT: ${ZABBIX_ENDPOINT} - ZABBIX_USER: ${ZABBIX_USER} - ZABBIX_PASSWORD: ${ZABBIX_PASSWORD} + ZABBIX_ENDPOINT: ${ZABBIX_ENDPOINT:-} + ZABBIX_USER: ${ZABBIX_USER:-} + ZABBIX_PASSWORD: ${ZABBIX_PASSWORD:-} - STALE_PERIOD: ${STALE_PERIOD} + STALE_PERIOD: ${STALE_PERIOD:-24} - PG_DB_HOST: ${PG_DB_HOST} - PG_DB_NAME: ${PG_DB_NAME} - PG_DB_USER: ${PG_DB_USER} - PG_DB_PASS: ${PG_DB_PASS} + PG_DB_HOST: ${PG_DB_HOST:-cvmanager_postgres:5432} + PG_DB_NAME: ${PG_DB_NAME:-postgres} + PG_DB_USER: ${PG_DB_USER:-postgres} + PG_DB_PASS: ${PG_DB_PASS:-postgres} - LOGGING_LEVEL: ${RSU_STATUS_LOGGING_LEVEL} + LOGGING_LEVEL: ${RSU_STATUS_LOGGING_LEVEL:-} logging: options: max-size: '10m' @@ -87,26 +86,26 @@ services: condition: service_healthy required: false environment: - STORAGE_TYPE: ${STORAGE_TYPE} + STORAGE_TYPE: ${STORAGE_TYPE:-postgres} - ISS_API_KEY: ${ISS_API_KEY} - ISS_API_KEY_NAME: ${ISS_API_KEY_NAME} - ISS_PROJECT_ID: ${ISS_PROJECT_ID} - ISS_SCMS_TOKEN_REST_ENDPOINT: ${ISS_SCMS_TOKEN_REST_ENDPOINT} - ISS_SCMS_VEHICLE_REST_ENDPOINT: ${ISS_SCMS_VEHICLE_REST_ENDPOINT} - ISS_KEY_TABLE_NAME: ${ISS_KEY_TABLE_NAME} + ISS_API_KEY: ${ISS_API_KEY:-} + ISS_API_KEY_NAME: ${ISS_API_KEY_NAME:-} + ISS_PROJECT_ID: ${ISS_PROJECT_ID:-} + ISS_SCMS_TOKEN_REST_ENDPOINT: ${ISS_SCMS_TOKEN_REST_ENDPOINT:-} + ISS_SCMS_VEHICLE_REST_ENDPOINT: ${ISS_SCMS_VEHICLE_REST_ENDPOINT:-} + ISS_KEY_TABLE_NAME: ${ISS_KEY_TABLE_NAME:-} - PG_DB_HOST: ${PG_DB_HOST} - PG_DB_NAME: ${PG_DB_NAME} - PG_DB_USER: ${PG_DB_USER} - PG_DB_PASS: ${PG_DB_PASS} + PG_DB_HOST: ${PG_DB_HOST:-cvmanager_postgres:5432} + PG_DB_NAME: ${PG_DB_NAME:-postgres} + PG_DB_USER: ${PG_DB_USER:-postgres} + PG_DB_PASS: ${PG_DB_PASS:-postgres} - PROJECT_ID: ${GCP_PROJECT_ID} + PROJECT_ID: ${GCP_PROJECT_ID:-} GOOGLE_APPLICATION_CREDENTIALS: /home/gcp_key.json - LOGGING_LEVEL: ${ISS_LOGGING_LEVEL} + LOGGING_LEVEL: ${ISS_LOGGING_LEVEL:-} volumes: - - ./resources/google/${GOOGLE_ACCESS_KEY_NAME}:/home/gcp_key.json + - ./resources/google/${GOOGLE_ACCESS_KEY_NAME:-}:/home/gcp_key.json logging: options: max-size: '10m' @@ -125,16 +124,16 @@ services: ports: - '8089:8080' environment: - PG_DB_HOST: ${PG_DB_HOST} - PG_DB_NAME: postgres - PG_DB_USER: ${PG_DB_USER} - PG_DB_PASS: ${PG_DB_PASS} + PG_DB_HOST: ${PG_DB_HOST:-cvmanager_postgres:5432} + PG_DB_NAME: ${PG_DB_NAME:-postgres} + PG_DB_USER: ${PG_DB_USER:-postgres} + PG_DB_PASS: ${PG_DB_PASS:-postgres} - UPGRADE_RUNNER_ENDPOINT: ${FIRMWARE_MANAGER_UPGRADE_RUNNER_ENDPOINT} + UPGRADE_RUNNER_ENDPOINT: ${FIRMWARE_MANAGER_UPGRADE_RUNNER_ENDPOINT:-http://firmware_manager_upgrade_runner:8090} - LOGGING_LEVEL: ${FIRMWARE_MANAGER_LOGGING_LEVEL} + LOGGING_LEVEL: ${FIRMWARE_MANAGER_LOGGING_LEVEL:-} volumes: - - ${HOST_BLOB_STORAGE_DIRECTORY}:/mnt/blob_storage + - ${HOST_BLOB_STORAGE_DIRECTORY:-./local_blob_storage}:/mnt/blob_storage logging: options: max-size: '10m' @@ -153,12 +152,12 @@ services: ports: - '8090:8080' environment: - BLOB_STORAGE_PROVIDER: ${BLOB_STORAGE_PROVIDER} - BLOB_STORAGE_BUCKET: ${BLOB_STORAGE_BUCKET} + BLOB_STORAGE_PROVIDER: ${BLOB_STORAGE_PROVIDER:-DOCKER} + BLOB_STORAGE_BUCKET: ${BLOB_STORAGE_BUCKET:-DOCKER} - FW_UPGRADE_MAX_RETRY_LIMIT: ${FW_UPGRADE_MAX_RETRY_LIMIT} + FW_UPGRADE_MAX_RETRY_LIMIT: ${FW_UPGRADE_MAX_RETRY_LIMIT:-3} - GCP_PROJECT: ${GCP_PROJECT_ID} + GCP_PROJECT: ${GCP_PROJECT_ID:-} GOOGLE_APPLICATION_CREDENTIALS: /home/gcp_key.json SMTP_SERVER_IP: ${SMTP_SERVER_IP} @@ -166,12 +165,12 @@ services: SMTP_USERNAME: ${SMTP_USERNAME} SMTP_PASSWORD: ${SMTP_PASSWORD} - UPGRADE_SCHEDULER_ENDPOINT: ${FIRMWARE_MANAGER_UPGRADE_SCHEDULER_ENDPOINT} + UPGRADE_SCHEDULER_ENDPOINT: ${FIRMWARE_MANAGER_UPGRADE_SCHEDULER_ENDPOINT:-http://firmware_manager_upgrade_scheduler:8090} - LOGGING_LEVEL: ${FIRMWARE_MANAGER_LOGGING_LEVEL} + LOGGING_LEVEL: ${FIRMWARE_MANAGER_LOGGING_LEVEL:-} volumes: - - ./resources/google/${GOOGLE_ACCESS_KEY_NAME}:/home/gcp_key.json - - ${HOST_BLOB_STORAGE_DIRECTORY}:/mnt/blob_storage + - ./resources/google/${GOOGLE_ACCESS_KEY_NAME:-sample_gcp_service_account.json}:/home/gcp_key.json + - ${HOST_BLOB_STORAGE_DIRECTORY:-./local_blob_storage}:/mnt/blob_storage logging: options: max-size: '10m' diff --git a/docker-compose-conflictmonitor.yml b/docker-compose-conflictmonitor.yml index 1e04dd42c..8ef75481e 100644 --- a/docker-compose-conflictmonitor.yml +++ b/docker-compose-conflictmonitor.yml @@ -8,14 +8,14 @@ services: - conflictmonitor - conflictmonitor_only image: ${DOCKERHUB_HOST:-usdotjpoode}/jpo-conflictmonitor:${DOCKERHUB_RELEASE:-2025-q2} - restart: ${RESTART_POLICY} + restart: ${RESTART_POLICY:-on-failure:3} ports: - '8082:8082' environment: - DOCKER_HOST_IP: ${DOCKER_HOST_IP:?error} - KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_BOOTSTRAP_SERVERS:?error} + DOCKER_HOST_IP: ${DOCKER_HOST_IP:-host.docker.internal} + KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} CONNECT_URL: ${CONNECT_URL:-http://kafka-connect:8083} - spring.kafka.bootstrap-servers: ${KAFKA_BOOTSTRAP_SERVERS:?error} + spring.kafka.bootstrap-servers: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} ROCKSDB_TOTAL_OFF_HEAP_MEMORY: ${ROCKSDB_TOTAL_OFF_HEAP_MEMORY:-134217728} ROCKSDB_INDEX_FILTER_BLOCK_RATIO: ${ROCKSDB_INDEX_FILTER_BLOCK_RATIO:-0.1} ROCKSDB_TOTAL_MEMTABLE_MEMORY: ${ROCKSDB_TOTAL_MEMTABLE_MEMORY:-67108864} @@ -47,7 +47,7 @@ services: - conflictmonitor - ode image: ${DOCKERHUB_HOST:-usdotjpoode}/jpo-ode:${DOCKERHUB_RELEASE:-2025-q2} - restart: ${RESTART_POLICY} + restart: ${RESTART_POLICY:-on-failure:3} ports: - '8080:8080' - '9090:9090' @@ -63,8 +63,8 @@ services: - '5555:5555/udp' - '6666:6666/udp' environment: - DOCKER_HOST_IP: ${DOCKER_HOST_IP:?error} - ODE_KAFKA_BROKERS: ${DOCKER_HOST_IP}:9092 + DOCKER_HOST_IP: ${DOCKER_HOST_IP:-host.docker.internal} + ODE_KAFKA_BROKERS: ${DOCKER_HOST_IP:-kafka}:9092 KAFKA_LINGER_MS: 1 KAFKA_ACKS: all KAFKA_RETRIES: 0 @@ -105,14 +105,14 @@ services: - conflictmonitor - adm image: ${DOCKERHUB_HOST:-usdotjpoode}/asn1_codec:${DOCKERHUB_RELEASE:-2025-q2} - restart: ${RESTART_POLICY} + restart: ${RESTART_POLICY:-on-failure:3} deploy: resources: limits: cpus: '1' memory: 2G environment: - DOCKER_HOST_IP: ${DOCKER_HOST_IP} + DOCKER_HOST_IP: ${DOCKER_HOST_IP:-host.docker.internal} ACM_CONFIG_FILE: adm.properties ACM_LOG_TO_CONSOLE: false ACM_LOG_TO_FILE: true @@ -131,14 +131,14 @@ services: - conflictmonitor - aem image: ${DOCKERHUB_HOST:-usdotjpoode}/asn1_codec:${DOCKERHUB_RELEASE:-2025-q2} - restart: ${RESTART_POLICY} + restart: ${RESTART_POLICY:-on-failure:3} deploy: resources: limits: cpus: '1' memory: 2G environment: - DOCKER_HOST_IP: ${DOCKER_HOST_IP} + DOCKER_HOST_IP: ${DOCKER_HOST_IP:-host.docker.internal} ACM_CONFIG_FILE: aem.properties ACM_LOG_TO_CONSOLE: false ACM_LOG_TO_FILE: true @@ -157,16 +157,16 @@ services: - conflictmonitor - geojsonconverter image: ${DOCKERHUB_HOST:-usdotjpoode}/geojsonconverter:${DOCKERHUB_RELEASE:-2025-q2} - restart: ${RESTART_POLICY} + restart: ${RESTART_POLICY:-on-failure:3} deploy: resources: limits: cpus: '1' memory: 2G environment: - DOCKER_HOST_IP: ${DOCKER_HOST_IP:?error} + DOCKER_HOST_IP: ${DOCKER_HOST_IP:-host.docker.internal} geometry.output.mode: GEOJSON_ONLY - spring.kafka.bootstrap-servers: ${DOCKER_HOST_IP:?error}:9092 + spring.kafka.bootstrap-servers: ${DOCKER_HOST_IP:-kafka}:9092 logging: options: max-size: '10m' @@ -182,13 +182,13 @@ services: - deduplicator image: ${DOCKERHUB_HOST:-usdotjpoode}/jpo-deduplicator:${DOCKERHUB_RELEASE:-2025-q2} privileged: false # Set true to allow writing to /proc/sys/vm/drop_caches - restart: ${RESTART_POLICY} + restart: ${RESTART_POLICY:-on-failure:3} ports: - '10091:10090' # JMX environment: - DOCKER_HOST_IP: ${DOCKER_HOST_IP} - KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_BOOTSTRAP_SERVERS} - SPRING_KAFKA_BOOTSTRAPSERVERS: ${KAFKA_BOOTSTRAP_SERVERS} + DOCKER_HOST_IP: ${DOCKER_HOST_IP:-host.docker.internal} + KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} + SPRING_KAFKA_BOOTSTRAPSERVERS: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} ENABLE_PROCESSED_MAP_DEDUPLICATION: true ENABLE_PROCESSED_MAP_WKT_DEDUPLICATION: true ENABLE_ODE_MAP_DEDUPLICATION: true diff --git a/docker-compose-intersection.yml b/docker-compose-intersection.yml index f727a8e21..1346a634e 100644 --- a/docker-compose-intersection.yml +++ b/docker-compose-intersection.yml @@ -13,43 +13,40 @@ services: dockerfile: Dockerfile.intersection-api args: MAVEN_GITHUB_TOKEN: ${MAVEN_GITHUB_TOKEN:?error} - MAVEN_GITHUB_ORG: ${MAVEN_GITHUB_ORG:?error} + MAVEN_GITHUB_ORG: ${MAVEN_GITHUB_ORG:-usdot-jpo-ode} ports: - '8089:8089' restart: always - extra_hosts: - ${WEBAPP_DOMAIN}: ${WEBAPP_HOST_IP} - ${KEYCLOAK_DOMAIN}: ${KC_HOST_IP} environment: - AUTH_SERVER_URL: ${KEYCLOAK_ENDPOINT} - DB_HOST_IP: ${DB_HOST_IP} - DB_HOST_PORT: ${DB_HOST_PORT} - SPRING_KAFKA_BOOTSTRAPSERVERS: ${KAFKA_BOOTSTRAP_SERVERS} - CM_SERVER_URL: ${CM_SERVER_URL} + AUTH_SERVER_URL: ${KEYCLOAK_ENDPOINT:-http://${DOCKER_HOST_IP:?error}:8084} + DB_HOST_IP: ${DB_HOST_IP:-mongo} + DB_HOST_PORT: ${DB_HOST_PORT:-27017} + SPRING_KAFKA_BOOTSTRAPSERVERS: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} + CM_SERVER_URL: ${CM_SERVER_URL:-http://conflictmonitor:8082} load: 'false' KAFKA_TYPE: 'ON-PREM' ACM_CONFIG_FILE: adm.properties ACM_LOG_TO_CONSOLE: true ACM_LOG_TO_FILE: false ACM_LOG_LEVEL: DEBUG - MONGO_READ_WRITE_USER: ${MONGO_READ_WRITE_USER} - MONGO_READ_WRITE_PASS: ${MONGO_READ_WRITE_PASS} - CM_MAXIMUM_RESPONSE_SIZE: ${CM_MAXIMUM_RESPONSE_SIZE} - SPRING_DATA_MONGODB_DATABASE: ${CM_DATABASE_NAME} + MONGO_READ_WRITE_USER: ${MONGO_READ_WRITE_USER:-ode} + MONGO_READ_WRITE_PASS: ${MONGO_READ_WRITE_PASS:-replace_me} + CM_MAXIMUM_RESPONSE_SIZE: ${CM_MAXIMUM_RESPONSE_SIZE:-10000} + SPRING_DATA_MONGODB_DATABASE: ${CM_DATABASE_NAME:-CV} - KEYCLOAK_CLIENT_API_ID: ${KEYCLOAK_API_CLIENT_ID} - KEYCLOAK_CLIENT_API_SECRET: ${KEYCLOAK_API_CLIENT_SECRET_KEY} + KEYCLOAK_CLIENT_API_ID: ${KEYCLOAK_API_CLIENT_ID:-cvmanager-api} + KEYCLOAK_CLIENT_API_SECRET: ${KEYCLOAK_API_CLIENT_SECRET_KEY:-keycloak-secret-key} - KAFKA_BROKER_IP: ${KAFKA_BROKER_IP} - KAFKA_BROKER_PORT: ${KAFKA_BROKER_PORT} + KAFKA_BROKER_IP: ${KAFKA_BROKER_IP:-kafka} + KAFKA_BROKER_PORT: ${KAFKA_BROKER_PORT:-5432} - PG_DB_USER: ${PG_DB_USER} - PG_DB_PASS: ${PG_DB_PASS} - POSTGRES_SERVER_URL: jdbc:postgresql://${PG_DB_HOST} + PG_DB_USER: ${PG_DB_USER:-postgres} + PG_DB_PASS: ${PG_DB_PASS:-postgres} + POSTGRES_SERVER_URL: jdbc:postgresql://${PG_DB_HOST:-cvmanager_postgres:5432} - INTERSECTION_API_ENABLE_API: ${INTERSECTION_API_ENABLE_API} - INTERSECTION_API_ENABLE_EMAILER: ${INTERSECTION_API_ENABLE_EMAILER} - INTERSECTION_API_ENABLE_REPORTS: ${INTERSECTION_API_ENABLE_REPORTS} + INTERSECTION_API_ENABLE_API: ${INTERSECTION_API_ENABLE_API:-True} + INTERSECTION_API_ENABLE_EMAILER: ${INTERSECTION_API_ENABLE_EMAILER:-false} + INTERSECTION_API_ENABLE_REPORTS: ${INTERSECTION_API_ENABLE_REPORTS:-false} BASE_ROUTE_PREFIX: ${INTERSECTION_API_ROUTE_PREFIX:-/} entrypoint: diff --git a/docker-compose-obu-ota-server.yml b/docker-compose-obu-ota-server.yml index f6c4b0f5a..d39f5dc82 100644 --- a/docker-compose-obu-ota-server.yml +++ b/docker-compose-obu-ota-server.yml @@ -12,22 +12,22 @@ services: ports: - 8085:8085 environment: - SERVER_HOST: ${OBU_OTA_SERVER_HOST} - LOGGING_LEVEL: ${OBU_OTA_LOGGING_LEVEL} - BLOB_STORAGE_PROVIDER: ${BLOB_STORAGE_PROVIDER} - BLOB_STORAGE_BUCKET: ${OBU_OTA_BLOB_STORAGE_BUCKET} - BLOB_STORAGE_PATH: ${OBU_OTA_BLOB_STORAGE_PATH} + SERVER_HOST: ${OBU_OTA_SERVER_HOST:-} + LOGGING_LEVEL: ${OBU_OTA_LOGGING_LEVEL:-} + BLOB_STORAGE_PROVIDER: ${BLOB_STORAGE_PROVIDER:-DOCKER} + BLOB_STORAGE_BUCKET: ${OBU_OTA_BLOB_STORAGE_BUCKET:-} + BLOB_STORAGE_PATH: ${OBU_OTA_BLOB_STORAGE_PATH:-} - OTA_USERNAME: ${OTA_USERNAME} - OTA_PASSWORD: ${OTA_PASSWORD} + OTA_USERNAME: ${OTA_USERNAME:-admin} + OTA_PASSWORD: ${OTA_PASSWORD:-admin} - PG_DB_HOST: ${PG_DB_HOST} - PG_DB_NAME: ${PG_DB_NAME} - PG_DB_USER: ${PG_DB_USER} - PG_DB_PASS: ${PG_DB_PASS} + PG_DB_HOST: ${PG_DB_HOST:-cvmanager_postgres} + PG_DB_USER: ${PG_DB_USER:-postgres} + PG_DB_PASS: ${PG_DB_PASS:-postgres} + PG_DB_NAME: ${PG_DB_NAME:-postgres} - MAX_COUNT: ${MAX_COUNT} - NGINX_ENCRYPTION: ${NGINX_ENCRYPTION} + MAX_COUNT: ${MAX_COUNT:-10} + NGINX_ENCRYPTION: ${NGINX_ENCRYPTION:-plain} volumes: - ./resources/ota/firmwares:/firmwares logging: @@ -48,12 +48,12 @@ services: - 443:443 environment: NGINX_ENVSUBST_OUTPUT_DIR: /etc/nginx - SERVER_HOST: ${OBU_OTA_SERVER_HOST} + SERVER_HOST: ${OBU_OTA_SERVER_HOST:-} volumes: - - ./resources/ota/nginx/nginx-${NGINX_ENCRYPTION}.conf:/etc/nginx/templates/nginx.conf.template + - ./resources/ota/nginx/nginx-${NGINX_ENCRYPTION:-plain}.conf:/etc/nginx/templates/nginx.conf.template - ./resources/ota/nginx/gen_dhparam.sh:/docker-entrypoint.d/gen_dhparam.sh - - ./resources/ota/nginx/ssl/${SERVER_CERT_FILE}:/etc/ssl/certs/ota_server.crt - - ./resources/ota/nginx/ssl/${SERVER_KEY_FILE}:/etc/ssl/private/ota_server.key + - ./resources/ota/nginx/ssl/${SERVER_CERT_FILE:-ota_server.crt}:/etc/ssl/certs/ota_server.crt + - ./resources/ota/nginx/ssl/${SERVER_KEY_FILE:-ota_server.key}:/etc/ssl/private/ota_server.key depends_on: - jpo_ota_backend logging: diff --git a/docker-compose-webapp-deployment.yml b/docker-compose-webapp-deployment.yml index 39fddf3b1..431b2e9f0 100644 --- a/docker-compose-webapp-deployment.yml +++ b/docker-compose-webapp-deployment.yml @@ -1,5 +1,4 @@ # This file is used to build the webapp image for deployment. -# The COUNTS_MSG_TYPES and DOT_NAME variables must be set in .env before building to populate # correctly in the deployed webapp as they are build-time variables. services: cvmanager_webapp: @@ -7,20 +6,19 @@ services: context: webapp dockerfile: Dockerfile args: - API_URI: ${API_ENDPOINT} - MAPBOX_TOKEN: ${MAPBOX_TOKEN} - KEYCLOAK_HOST_URL: ${KEYCLOAK_ENDPOINT} - COUNT_MESSAGE_TYPES: ${COUNTS_MSG_TYPES} - VIEWER_MESSAGE_TYPES: ${VIEWER_MSG_TYPES} - DOT_NAME: ${DOT_NAME} - MAPBOX_INIT_LATITUDE: ${MAPBOX_INIT_LATITUDE} - MAPBOX_INIT_LONGITUDE: ${MAPBOX_INIT_LONGITUDE} - MAPBOX_INIT_ZOOM: ${MAPBOX_INIT_ZOOM} + API_URI: ${API_ENDPOINT:?error} + MAPBOX_TOKEN: ${MAPBOX_TOKEN:?error} + KEYCLOAK_HOST_URL: ${KEYCLOAK_ENDPOINT:?error} + VIEWER_MESSAGE_TYPES: ${VIEWER_MSG_TYPES:-BSM} + DOT_NAME: ${DOT_NAME:-CDOT} + MAPBOX_INIT_LATITUDE: ${MAPBOX_INIT_LATITUDE:-39.7392} + MAPBOX_INIT_LONGITUDE: ${MAPBOX_INIT_LONGITUDE:--104.9903} + MAPBOX_INIT_ZOOM: ${MAPBOX_INIT_ZOOM:-10} CVIZ_API_SERVER_URL: ${CVIZ_API_SERVER_URL} CVIZ_API_WS_URL: ${CVIZ_API_WS_URL} - ENABLE_RSU_FEATURES: ${ENABLE_RSU_FEATURES} - ENABLE_INTERSECTION_FEATURES: ${ENABLE_INTERSECTION_FEATURES} - ENABLE_WZDX_FEATURES: ${ENABLE_WZDX_FEATURES} + ENABLE_RSU_FEATURES: ${ENABLE_RSU_FEATURES:-True} + ENABLE_INTERSECTION_FEATURES: ${ENABLE_INTERSECTION_FEATURES:-True} + ENABLE_WZDX_FEATURES: ${ENABLE_WZDX_FEATURES:-True} image: jpo_cvmanager_webapp:latest restart: always logging: diff --git a/docker-compose.yml b/docker-compose.yml index bd06227bc..d18c685d9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,27 +14,22 @@ services: dockerfile: Dockerfile.api image: jpo_cvmanager_api:latest restart: always - extra_hosts: - ${WEBAPP_DOMAIN}: ${WEBAPP_HOST_IP} - ${KEYCLOAK_DOMAIN}: ${KC_HOST_IP} ports: - '8081:5000' environment: - PG_DB_HOST: ${PG_DB_HOST} - PG_DB_USER: ${PG_DB_USER} - PG_DB_PASS: ${PG_DB_PASS} - PG_DB_NAME: postgres + PG_DB_HOST: ${PG_DB_HOST:-cvmanager_postgres:5432} + PG_DB_USER: ${PG_DB_USER:-postgres} + PG_DB_PASS: ${PG_DB_PASS:-postgres} + PG_DB_NAME: ${PG_DB_NAME:-postgres} INSTANCE_CONNECTION_NAME: ${INSTANCE_CONNECTION_NAME} - MONGO_DB_URI: ${MONGO_DB_URI} - MONGO_DB_NAME: ${CM_DATABASE_NAME} - MONGO_PROCESSED_BSM_COLLECTION_NAME: ${MONGO_PROCESSED_BSM_COLLECTION_NAME} - MONGO_PROCESSED_PSM_COLLECTION_NAME: ${MONGO_PROCESSED_PSM_COLLECTION_NAME} + MONGO_DB_URI: ${MONGO_DB_URI:-mongodb://${MONGO_READ_WRITE_USER:-ode}:${MONGO_READ_WRITE_PASS:-replace-me}@${DB_HOST_IP:-mongo}:${DB_HOST_PORT:-27017}/?directConnection=true&authSource=admin} + MONGO_DB_NAME: ${CM_DATABASE_NAME:-CV} + MONGO_PROCESSED_BSM_COLLECTION_NAME: ${MONGO_PROCESSED_BSM_COLLECTION_NAME:-processed_bsm} + MONGO_PROCESSED_PSM_COLLECTION_NAME: ${MONGO_PROCESSED_PSM_COLLECTION_NAME:-processed_psm} - COUNTS_MSG_TYPES: ${COUNTS_MSG_TYPES} - - SSM_DB_NAME: ${SSM_DB_NAME} - SRM_DB_NAME: ${SRM_DB_NAME} + MONGO_SSM_COLLECTION_NAME: ${MONGO_SSM_COLLECTION_NAME} + MONGO_SRM_COLLECTION_NAME: ${MONGO_SRM_COLLECTION_NAME} MAX_GEO_QUERY_RECORDS: ${MAX_GEO_QUERY_RECORDS} @@ -43,36 +38,36 @@ services: WZDX_API_KEY: ${WZDX_API_KEY} WZDX_ENDPOINT: ${WZDX_ENDPOINT} - ENABLE_RSU_FEATURES: ${ENABLE_RSU_FEATURES} - ENABLE_INTERSECTION_FEATURES: ${ENABLE_INTERSECTION_FEATURES} - ENABLE_WZDX_FEATURES: ${ENABLE_WZDX_FEATURES} - ENABLE_MOOVE_AI_FEATURES: ${ENABLE_MOOVE_AI_FEATURES} - - CORS_DOMAIN: ${CORS_DOMAIN} - KEYCLOAK_ENDPOINT: ${KEYCLOAK_ENDPOINT} - KEYCLOAK_REALM: ${KEYCLOAK_REALM} - KEYCLOAK_API_CLIENT_ID: ${KEYCLOAK_API_CLIENT_ID} - KEYCLOAK_API_CLIENT_SECRET_KEY: ${KEYCLOAK_API_CLIENT_SECRET_KEY} - - CSM_EMAIL_TO_SEND_FROM: ${CSM_EMAIL_TO_SEND_FROM} - CSM_EMAILS_TO_SEND_TO: ${CSM_EMAILS_TO_SEND_TO} - CSM_TARGET_SMTP_SERVER_ADDRESS: ${CSM_TARGET_SMTP_SERVER_ADDRESS} - CSM_TARGET_SMTP_SERVER_PORT: ${CSM_TARGET_SMTP_SERVER_PORT} - CSM_TLS_ENABLED: ${CSM_TLS_ENABLED} - CSM_AUTH_ENABLED: ${CSM_AUTH_ENABLED} - CSM_EMAIL_APP_USERNAME: ${CSM_EMAIL_APP_USERNAME} - CSM_EMAIL_APP_PASSWORD: ${CSM_EMAIL_APP_PASSWORD} - - ENVIRONMENT_NAME: ${ENVIRONMENT_NAME} - LOGS_LINK: ${LOGS_LINK} + ENABLE_RSU_FEATURES: ${ENABLE_RSU_FEATURES:-true} + ENABLE_INTERSECTION_FEATURES: ${ENABLE_INTERSECTION_FEATURES:-true} + ENABLE_WZDX_FEATURES: ${ENABLE_WZDX_FEATURES:-false} + ENABLE_MOOVE_AI_FEATURES: ${ENABLE_MOOVE_AI_FEATURES:-false} + + CORS_DOMAIN: ${CORS_DOMAIN:-*} + KEYCLOAK_ENDPOINT: ${KEYCLOAK_ENDPOINT:-http://${DOCKER_HOST_IP:?error}:8084} + KEYCLOAK_REALM: ${KEYCLOAK_REALM:-cvmanager} + KEYCLOAK_API_CLIENT_ID: ${KEYCLOAK_API_CLIENT_ID:-cvmanager-api} + KEYCLOAK_API_CLIENT_SECRET_KEY: ${KEYCLOAK_API_CLIENT_SECRET_KEY:-keycloak-secret-key} + + CSM_EMAIL_TO_SEND_FROM: ${CSM_EMAIL_TO_SEND_FROM:-} + CSM_EMAILS_TO_SEND_TO: ${CSM_EMAILS_TO_SEND_TO:-} + CSM_TARGET_SMTP_SERVER_ADDRESS: ${CSM_TARGET_SMTP_SERVER_ADDRESS:-} + CSM_TARGET_SMTP_SERVER_PORT: ${CSM_TARGET_SMTP_SERVER_PORT:-} + CSM_TLS_ENABLED: ${CSM_TLS_ENABLED:-} + CSM_AUTH_ENABLED: ${CSM_AUTH_ENABLED:-} + CSM_EMAIL_APP_USERNAME: ${CSM_EMAIL_APP_USERNAME:-} + CSM_EMAIL_APP_PASSWORD: ${CSM_EMAIL_APP_PASSWORD:-} + + ENVIRONMENT_NAME: ${ENVIRONMENT_NAME:-} + LOGS_LINK: ${LOGS_LINK:-} GOOGLE_APPLICATION_CREDENTIALS: /home/gcp_key.json - GCP_PROJECT_ID: ${GCP_PROJECT_ID} - MOOVE_AI_SEGMENT_AGG_STATS_TABLE: ${MOOVE_AI_SEGMENT_AGG_STATS_TABLE} - MOOVE_AI_SEGMENT_EVENT_STATS_TABLE: ${MOOVE_AI_SEGMENT_EVENT_STATS_TABLE} + GCP_PROJECT_ID: ${GCP_PROJECT_ID:-} + MOOVE_AI_SEGMENT_AGG_STATS_TABLE: ${MOOVE_AI_SEGMENT_AGG_STATS_TABLE:-} + MOOVE_AI_SEGMENT_EVENT_STATS_TABLE: ${MOOVE_AI_SEGMENT_EVENT_STATS_TABLE:-} - TIMEZONE: ${TIMEZONE} - LOGGING_LEVEL: ${API_LOGGING_LEVEL} + TIMEZONE: ${TIMEZONE:-} + LOGGING_LEVEL: ${API_LOGGING_LEVEL:-} volumes: - ./resources/google/${GOOGLE_ACCESS_KEY_NAME}:/home/gcp_key.json logging: @@ -88,26 +83,25 @@ services: context: webapp dockerfile: Dockerfile args: - API_URI: ${API_ENDPOINT} - MAPBOX_TOKEN: ${MAPBOX_TOKEN} - KEYCLOAK_HOST_URL: ${KEYCLOAK_ENDPOINT} - KEYCLOAK_REALM: ${KEYCLOAK_REALM} - KEYCLOAK_CLIENT_ID: ${KEYCLOAK_GUI_CLIENT_ID} - COUNT_MESSAGE_TYPES: ${COUNTS_MSG_TYPES} - VIEWER_MESSAGE_TYPES: ${VIEWER_MSG_TYPES} - DOT_NAME: ${DOT_NAME} - MAPBOX_INIT_LATITUDE: ${MAPBOX_INIT_LATITUDE} - MAPBOX_INIT_LONGITUDE: ${MAPBOX_INIT_LONGITUDE} - MAPBOX_INIT_ZOOM: ${MAPBOX_INIT_ZOOM} - CVIZ_API_SERVER_URL: ${CVIZ_API_SERVER_URL}${INTERSECTION_API_ROUTE_PREFIX:-/} - CVIZ_API_WS_URL: ${CVIZ_API_WS_URL}${INTERSECTION_API_ROUTE_PREFIX:-/} - ENABLE_RSU_FEATURES: ${ENABLE_RSU_FEATURES} - ENABLE_INTERSECTION_FEATURES: ${ENABLE_INTERSECTION_FEATURES} - ENABLE_WZDX_FEATURES: ${ENABLE_WZDX_FEATURES} - ENABLE_MOOVE_AI_FEATURES: ${ENABLE_MOOVE_AI_FEATURES} - WEBAPP_THEME_LIGHT: ${WEBAPP_THEME_LIGHT} - WEBAPP_THEME_DARK: ${WEBAPP_THEME_DARK} - ENABLE_HAAS_FEATURES: ${ENABLE_HAAS_FEATURES} + API_URI: ${API_ENDPOINT:-http://localhost:8081} + MAPBOX_TOKEN: ${MAPBOX_TOKEN?:error} + KEYCLOAK_HOST_URL: ${KEYCLOAK_ENDPOINT:-http://${DOCKER_HOST_IP:?error}:8084} + KEYCLOAK_REALM: ${KEYCLOAK_REALM:-cvmanager} + KEYCLOAK_CLIENT_ID: ${KEYCLOAK_GUI_CLIENT_ID:-cvmanager-gui} + VIEWER_MESSAGE_TYPES: ${VIEWER_MSG_TYPES:-BSM} + DOT_NAME: ${DOT_NAME:-CDOT} + MAPBOX_INIT_LATITUDE: ${MAPBOX_INIT_LATITUDE:-39.7392} + MAPBOX_INIT_LONGITUDE: ${MAPBOX_INIT_LONGITUDE:--104.9903} + MAPBOX_INIT_ZOOM: ${MAPBOX_INIT_ZOOM:-10} + CVIZ_API_SERVER_URL: ${CVIZ_API_SERVER_URL:-http://localhost:8089}${INTERSECTION_API_ROUTE_PREFIX:-/} + CVIZ_API_WS_URL: ${CVIZ_API_WS_URL:-ws://localhost:8089}${INTERSECTION_API_ROUTE_PREFIX:-/} + ENABLE_RSU_FEATURES: ${ENABLE_RSU_FEATURES:-true} + ENABLE_INTERSECTION_FEATURES: ${ENABLE_INTERSECTION_FEATURES:-true} + ENABLE_WZDX_FEATURES: ${ENABLE_WZDX_FEATURES:-false} + ENABLE_MOOVE_AI_FEATURES: ${ENABLE_MOOVE_AI_FEATURES:-false} + ENABLE_HAAS_FEATURES: ${ENABLE_HAAS_FEATURES:-true} + WEBAPP_THEME_LIGHT: ${WEBAPP_THEME_LIGHT:-dark} + WEBAPP_THEME_DARK: ${WEBAPP_THEME_DARK:-dark} image: jpo_cvmanager_webapp:latest restart: always volumes: @@ -117,11 +111,8 @@ services: cvmanager_keycloak: condition: service_healthy required: false - extra_hosts: - ${WEBAPP_DOMAIN}: ${WEBAPP_HOST_IP} - ${KEYCLOAK_DOMAIN}: ${KC_HOST_IP} ports: - - '80:80' + - '${WEBAPP_PORT:-3000}:80' logging: options: max-size: '10m' @@ -130,14 +121,13 @@ services: profiles: - basic - cvmanager_postgres - - intersection_no_api image: postgis/postgis:15-master restart: always ports: - '5432:5432' environment: - POSTGRES_USER: ${PG_DB_USER} - POSTGRES_PASSWORD: ${PG_DB_PASS} + POSTGRES_USER: ${PG_DB_USER:-postgres} + POSTGRES_PASSWORD: ${PG_DB_PASS:-postgres} volumes: - pgdb:/var/lib/postgresql/data - ./resources/sql_scripts:/docker-entrypoint-initdb.d @@ -150,43 +140,39 @@ services: - basic - keycloak - cvmanager_keycloak - - intersection_no_api build: context: ./resources/keycloak dockerfile: Dockerfile args: - KEYCLOAK_LOGIN_THEME_NAME: ${KEYCLOAK_LOGIN_THEME_NAME}.jar + KEYCLOAK_LOGIN_THEME_NAME: ${KEYCLOAK_LOGIN_THEME_NAME:-sample_theme}.jar image: jpo_cvmanager_keycloak:latest restart: always depends_on: cvmanager_postgres: required: false condition: service_started - - extra_hosts: - ${WEBAPP_DOMAIN}: ${WEBAPP_HOST_IP} - ${KEYCLOAK_DOMAIN}: ${KC_HOST_IP} ports: - '8084:8080' - '9000:9000' environment: - KEYCLOAK_ADMIN: ${KEYCLOAK_ADMIN} - KEYCLOAK_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD} - WEBAPP_ORIGIN: ${WEBAPP_ENDPOINT} - KC_BOOTSTRAP_ADMIN_USERNAME: ${KEYCLOAK_ADMIN} - KC_BOOTSTRAP_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD} + KEYCLOAK_ADMIN: ${KEYCLOAK_ADMIN:-admin} + KEYCLOAK_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD:-admin} + WEBAPP_ENDPOINT: ${WEBAPP_ENDPOINT:-http://localhost:${WEBAPP_PORT:-3000}} + KC_BOOTSTRAP_ADMIN_USERNAME: ${KEYCLOAK_ADMIN:-admin} + KC_BOOTSTRAP_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD:-admin} KC_HEALTH_ENABLED: 'true' - KC_LOG_LEVEL: ${KC_LOGGING_LEVEL} + KC_LOG_LEVEL: ${KC_LOG_LEVEL:-INFO} KC_DB: postgres - KC_DB_URL: jdbc:postgresql://${PG_DB_HOST}/postgres?currentSchema=keycloak - KC_DB_USERNAME: ${PG_DB_USER} - KC_DB_PASSWORD: ${PG_DB_PASS} - KC_HOSTNAME: ${KEYCLOAK_DOMAIN} - KEYCLOAK_API_CLIENT_SECRET_KEY: ${KEYCLOAK_API_CLIENT_SECRET_KEY} - GOOGLE_CLIENT_ID: ${GOOGLE_CLIENT_ID} - GOOGLE_CLIENT_SECRET: ${GOOGLE_CLIENT_SECRET} - KC_HOSTNAME_STRICT_HTTPS: 'false' - KC_HTTP_ENABLED: 'true' + KC_DB_SCHEMA: keycloak + KC_DB_URL_HOST: cvmanager_postgres + KC_DB_URL_DATABASE: postgres + KC_DB_URL_PORT: 5432 + KC_DB_USERNAME: ${PG_DB_USER:-postgres} + KC_DB_PASSWORD: ${PG_DB_PASS:-postgres} + KC_HOSTNAME: ${KEYCLOAK_ENDPOINT:-http://${DOCKER_HOST_IP:?error}:8084} + KEYCLOAK_API_CLIENT_SECRET_KEY: ${KEYCLOAK_API_CLIENT_SECRET_KEY:-keycloak-secret-key} + GOOGLE_CLIENT_ID: ${GOOGLE_CLIENT_ID:-} + GOOGLE_CLIENT_SECRET: ${GOOGLE_CLIENT_SECRET:-} command: - start-dev - --import-realm diff --git a/docs/debugging/login_unsuccessful_unknown_error_occurred.png b/docs/debugging/login_unsuccessful_unknown_error_occurred.png new file mode 100644 index 000000000..4b4584516 Binary files /dev/null and b/docs/debugging/login_unsuccessful_unknown_error_occurred.png differ diff --git a/docs/developer_best_practices.md b/docs/developer_best_practices.md new file mode 100644 index 000000000..a1cce80b9 --- /dev/null +++ b/docs/developer_best_practices.md @@ -0,0 +1,96 @@ +# CV Manager Developer Best Practices + +## Managing Fork Synchronization + +The majority of the CV-Manager development is completed on forks. This process enables development within a controlled environment. One major consideration is how often to synchronize with the upstream repository. The current recommended approach is to develop features within a fork, and contribute/push sets of features to the upstream repository (in this case, USDOT). + +### Upstream ahead of current + +When changes exist on the upstream repository which are not present on the current repository, the "Sync Fork" button can be used to bring those changes into the downstream repository. This can show 3 different menus depending on the diff: + +#### No changes present + +Sync Fork: No Changes + +1. No actions necessary, fork is in sync with upstream + +#### Changes present, no conflicts + +Sync Fork: Update branch | Discard N commits + +- Changes can be merged without review +- If you press "Update Branch", GitHub will merge the upstream commits into the current repository immediately, without creating a PR. This is the preferred approach +- If you would like to create a PR instead, please see [Changes present, conflicts](#changes-present-conflicts) +- WARNING: If you press "Discard N commits", all local commits not present on the upstream repository will be removed. This process is instant and without additional confirmation. See the steps below for completing this process safely + + 1. Create a copy of the current branch (henceforth assumed to be develop). The naming convention is "history/2025_q3" for a major release, or "history/2025_12_31" for date-based + 2. Navigate back to the develop branch, hit "Sync fork", and hit "Discard N commits" + 3. Clone the repository. If the repository is already cloned, checkout the develop branch (the one you discarded the commits on) and run the following command (swap out develop if the branch has a different name) + + ```sh + git reset --hard origin/develop + ``` + + 4. Copy the history branch to a new branch (named something like "develop-rebase-2025_12_31") + 5. Rebase develop into your new branch + ``` + git rebase develop + ``` + 6. Resolve merge conflicts + 7. Create a PR to merge the new branch changes into the default branch + +#### Changes present, conflicts + +Sync Fork: Open pull request | Discard N commits + +- Github has detected that the upstream branch cannot be merged into the current branch without conflicts. The PR that it offers to create is from the current branch into the upstream repository, which is the opposite direction of what we want. We want to resolve the conflicts on our fork, then push up the cleaned up changes at a later date. See the next bullet for instructions on how to create a PR from the upstream branch to the current/default branch +- To create a PR from the upstream repo to yours, fill in the following url: + - https://github.com/{your-organization or user}/{repo name}/compare/{default branch}...{upstream org name}:{repo name}:{default branch} + - Example: https://github.com/cdot-cv/jpo-cvmanager/compare/develop...usdot-jpo-ode:jpo-cvmanager:develop +- See the section above for a description of the "Discard N commits" button function + +## Pull Requests + +Pull requests should be kept to a manageable size, able to be reviewed within 1-2 hours. An ideal PR should be under 400 lines of code changed, with an upper limit of 1000 lines of code changed (using the guideline of 500 lines per hour). If a PR exceeds this size, consider breaking it up into multiple smaller PRs. Exceptions can be made for lines which are auto-generated. + +For more information on PR best practices, see [best-practices-for-peer-code-review](https://smartbear.com/learn/code-review/best-practices-for-peer-code-review) + +When creating a pull request, use the provided [pull request template](../pull_request_template.md). This ensures that all necessary information is provided for reviewers. + +### Squash Merge + +All pull requests should be squash merged into develop. This ensures that we have a history of feature additions without the noise of intermediate commits. + +When making a squash merge, both a message and a description are included. The message should be concise and accurate (this is what is seen first when scrolling through previous commits). This should resemble the PR title. +The extended description should describe important details about the feature including the distinct changes involved and the affected services. This should resemble the PR description. + +#### Synchronizing after a squash commit + +When a squash merge or squash commit is executed, multiple previous commits are replaced by a single squash commit. For any branches which still have the non-squished commits, they need to be synchronized. + +Checkout develop and check for pending changes. + +```sh +git checkout develop +git status +``` + +Ensure there are no pending changes. If there are any changes present, commit/stash them on a feature branch. The next command will delete all pending changes and re-set the history of your local develop to the remote develop + +```sh +git reset --hard origin/develop +``` + +The recommended approach for synchronizing feature branches after a squash is a [merge](https://git-scm.com/book/en/v2/Git-Branching-Basic-Branching-and-Merging). This will pull commits from the source branch into the feature branch, _retaining the un-squashed commits_, while presenting all merge conflicts in 1 wave. These additional un-squashed commits will be removed on the next squash merge into develop. + +```sh +git checkout feature-branch +git merge origin/develop +``` + +If removing the un-squashed commits is a priority, consider using a [rebase](https://git-scm.com/book/ms/v2/Git-Branching-Rebasing). This will _remove all of the un-squashed commits_, then go through the new commits and re-apply them to the updated history. Each step can have it's own merge conflicts, which can be extremely burdensome for large offsets, and each of the new commits will be re-applied, which will change their commit hashes. This can be a risky process, so ensure that the feature branch is backed up before proceeding. + +```sh +git checkout feature-branch +git rebase develop +``` diff --git a/docs/pr_screenshots/sync_fork_no_changes.png b/docs/pr_screenshots/sync_fork_no_changes.png new file mode 100644 index 000000000..62ed570ab Binary files /dev/null and b/docs/pr_screenshots/sync_fork_no_changes.png differ diff --git a/docs/pr_screenshots/sync_fork_open_pr.png b/docs/pr_screenshots/sync_fork_open_pr.png new file mode 100644 index 000000000..b34a19225 Binary files /dev/null and b/docs/pr_screenshots/sync_fork_open_pr.png differ diff --git a/docs/pr_screenshots/sync_fork_update_branch.png b/docs/pr_screenshots/sync_fork_update_branch.png new file mode 100644 index 000000000..ea29aa298 Binary files /dev/null and b/docs/pr_screenshots/sync_fork_update_branch.png differ diff --git a/jpo-utils b/jpo-utils index d9298120d..71625a752 160000 --- a/jpo-utils +++ b/jpo-utils @@ -1 +1 @@ -Subproject commit d9298120d603b9ff9ce3a5520ce885c8bd7ec23d +Subproject commit 71625a75254edba7f2af70a7267d7fda70cc4b01 diff --git a/resources/keycloak/realm.json b/resources/keycloak/realm.json index 36fe2dfdd..d2a090846 100644 --- a/resources/keycloak/realm.json +++ b/resources/keycloak/realm.json @@ -688,8 +688,20 @@ "enabled": true, "alwaysDisplayInConsole": false, "clientAuthenticatorType": "client-secret", - "redirectUris": ["http://localhost:3000/*", "http://localhost/*", "http://cvmanager.local.com/*"], - "webOrigins": ["http://localhost:3000", "http://localhost", "http://cvmanager.local.com"], + "redirectUris": [ + "http://localhost:3000/*", + "http://localhost:3001/*", + "http://localhost:3002/*", + "http://localhost/*", + "${WEBAPP_ENDPOINT}/*" + ], + "webOrigins": [ + "http://localhost:3000", + "http://localhost:3001", + "http://localhost:3002", + "http://localhost", + "${WEBAPP_ENDPOINT}" + ], "notBefore": 0, "bearerOnly": false, "consentRequired": false, @@ -1588,7 +1600,9 @@ "providerId": "custom-user-provider", "subComponents": {}, "config": { - "JDBC_URL": ["${KC_DB_URL}"], + "JDBC_URL": [ + "jdbc:postgresql://${KC_DB_URL_HOST}:${KC_DB_URL_PORT}/${KC_DB_URL_DATABASE}?currentSchema=${KC_DB_SCHEMA}" + ], "DB_USERNAME": ["${KC_DB_USERNAME}"], "VALIDATION_QUERY": ["select 1"], "cachePolicy": ["NO_CACHE"], diff --git a/resources/keycloak/sample_theme.jar b/resources/keycloak/sample_theme.jar index b8a01cc87..5bd0af813 100644 Binary files a/resources/keycloak/sample_theme.jar and b/resources/keycloak/sample_theme.jar differ diff --git a/resources/kubernetes/cv-manager-api.yaml b/resources/kubernetes/cv-manager-api.yaml index 11a2f0192..433c1dfb7 100644 --- a/resources/kubernetes/cv-manager-api.yaml +++ b/resources/kubernetes/cv-manager-api.yaml @@ -132,15 +132,13 @@ spec: value: "" - name: MONGO_DB_NAME value: "" - - name: COUNTS_MSG_TYPES - value: "" - name: MONGO_PROCESSED_BSM_COLLECTION_NAME value: "" - name: MONGO_PROCESSED_PSM_COLLECTION_NAME value: "" - - name: SSM_DB_NAME + - name: MONGO_SSM_COLLECTION_NAME value: "" - - name: SRM_DB_NAME + - name: MONGO_SRM_COLLECTION_NAME value: "" - name: WZDX_ENDPOINT value: "" diff --git a/resources/kubernetes/rsu-status-checker.yaml b/resources/kubernetes/rsu-status-checker.yaml index f18d0c467..8567154ff 100644 --- a/resources/kubernetes/rsu-status-checker.yaml +++ b/resources/kubernetes/rsu-status-checker.yaml @@ -35,7 +35,7 @@ spec: value: - name: ZABBIX value: - - name: RSU_SNMP_FETCH + - name: RSU_MSGFWD_FETCH value: - name: PG_DB_HOST value: diff --git a/sample-full.env b/sample-full.env new file mode 100644 index 000000000..078969b73 --- /dev/null +++ b/sample-full.env @@ -0,0 +1,459 @@ +######## ---------------------- DOCKER COMPOSE PROFILES ---------------------- ######## + +# Compose Profiles - see [README](README.md#docker-profiles) and sections below for more information +# There are a number of profiles available to start up groups of services. +# Additionally, each individual service in this project can be started by specifying its service name as a profile. +# The currently available profile groups are listed below. +# basic, webapp, mongo_full, kafka_full, intersection, intersection_no_api, conflictmonitor, addons, obu_ota, kafka_connect_standalone +COMPOSE_PROFILES=basic,webapp,mongo_full,kafka_full,intersection,kafka_connect_standalone + + +######## -------- Required Variables +# run 'ifconfig' in wsl +DOCKER_HOST_IP= + +# Mapbox token for map rendering in the webapp +MAPBOX_TOKEN= + +# GitHub Token (Required for Intersection API) - See services/intersection-api/README.md#github-token for steps to generate +MAVEN_GITHUB_TOKEN= + + +######## -------- General Variables - Apply to All Profiles +KEYCLOAK_DOMAIN=${DOCKER_HOST_IP} +KEYCLOAK_ENDPOINT=http://${KEYCLOAK_DOMAIN}:8084 + +WEBAPP_DOMAIN=localhost +WEBAPP_PORT=3000 +WEBAPP_ENDPOINT=http://${WEBAPP_DOMAIN}:${WEBAPP_PORT} + +API_DOMAIN=${DOCKER_HOST_IP} +API_ENDPOINT=http://${API_DOMAIN}:8081 + +# Remote dockerhub variables for ODE and ConflictMonitor images +DOCKERHUB_HOST=usdotjpoode +DOCKERHUB_RELEASE=2025-q2 + +# Logging Levels - "DEBUG", "INFO", "WARNING", "ERROR" +API_LOGGING_LEVEL="INFO" +FIRMWARE_MANAGER_LOGGING_LEVEL="INFO" +GEO_LOGGING_LEVEL="INFO" +ISS_LOGGING_LEVEL="INFO" +RSU_STATUS_LOGGING_LEVEL="INFO" +COUNTS_LOGGING_LEVEL="INFO" +OBU_OTA_LOGGING_LEVEL="INFO" + +# Also includes "ALL", "FATAL", "OFF", "TRACE" and "WARN" +KC_LOG_LEVEL="INFO" + +# Feature Flags +ENABLE_RSU_FEATURES='true' # 'false' to disable +ENABLE_INTERSECTION_FEATURES='true' # 'false' to disable +ENABLE_WZDX_FEATURES='true' # 'false' to disable +ENABLE_MOOVE_AI_FEATURES='true' # 'false' to disable +ENABLE_HAAS_FEATURES='true' # 'false' to disable + + +######## -------- "basic" Docker Profile Services +# Run critical cvmanager services +# Requires: None +# Compose file: docker-compose.yml +# Services: +# - cvmanager_api +# - Python backend api for webapp +# - cvmanager_postgres +# - Postgres database for cvmanager data and backing database for keycloak instance +# - cvmanager_keycloak +# - Keycloak instance for user authentication and authorization of webapp and api requests + +#### ---- cvmanager_keycloak +# Keycloak authentication credentials +KEYCLOAK_ADMIN=admin +KEYCLOAK_ADMIN_PASSWORD=admin + +# Keycloak Parameters - to generate secret key use a password generator such as: https://www.avast.com/en-us/random-password-generator#pc and set the length to 32 +KEYCLOAK_REALM=cvmanager +KEYCLOAK_GUI_CLIENT_ID=cvmanager-gui +KEYCLOAK_API_CLIENT_ID=cvmanager-api +KEYCLOAK_API_CLIENT_SECRET_KEY=w8zpoArUwIVN6TSDY5WQgX9TlVAgH9OF +KEYCLOAK_LOGIN_THEME_NAME=sample_theme + +# GCP OAuth2.0 client ID for SSO authentication in keycloak - if not specified the google SSO will not be functional +GOOGLE_CLIENT_ID= +GOOGLE_CLIENT_SECRET= + +#### ---- cvmanager_postgres + +# PostgreSQL Database connection information +# this value may need to follow with the webapp host if debugging the applications +PG_DB_HOST=${DOCKER_HOST_IP}:5432 +PG_DB_NAME=postgres +PG_DB_USER=postgres +# If the PG_DB_PASS variable has special characters, make sure to wrap it in single quotes +PG_DB_PASS=postgres + +#### ---- cvmanager_api + +# Allowed CORS domain for accessing the CV Manager API from (set to the web application hostname) +# Make sure to include http:// or https:// +# If using docker then this value should be set to: http://${WEBAPP_HOST_IP}:3000 +# If running the webapp using npm then set it to: http://localhost:3000 +# Leave as * to allow all domains access +CORS_DOMAIN=* + +# Set these variables if using either "MONGODB" or "BIGQUERY" for COUNT_DESTINATION_DB of jpo_count_metric +MONGO_PROCESSED_BSM_COLLECTION_NAME='ProcessedBsm' +MONGO_PROCESSED_PSM_COLLECTION_NAME='ProcessedPsm' +MONGO_SSM_COLLECTION_NAME= +MONGO_SRM_COLLECTION_NAME= + +# Specifies the maximum number of V2X messages returned from the geo_query_geo_data_mongo method before filtering occurs +MAX_GEO_QUERY_RECORDS= + +# If running firmware manager addon +FIRMWARE_MANAGER_ENDPOINT=http://${DOCKER_HOST_IP}:8089 + +# If connecting to PGDB over websocket: +INSTANCE_CONNECTION_NAME= + +# Python timezone for the CV Manager (You can list pytz timezones with the command 'pytz.all_timezones') +TIMEZONE="US/Mountain" + +# WZDx API key and endpoint for pulling WZDx data into the CV Manager +WZDX_API_KEY= +WZDX_ENDPOINT=data.cotrip.org + +# Contact Support Menu Email Configuration +CSM_EMAIL_TO_SEND_FROM= +CSM_EMAIL_APP_USERNAME= +CSM_EMAIL_APP_PASSWORD= +CSM_EMAILS_TO_SEND_TO= +CSM_TARGET_SMTP_SERVER_ADDRESS= +CSM_TARGET_SMTP_SERVER_PORT=587 +CSM_TLS_ENABLED=true +CSM_AUTH_ENABLED=true + +# Error Email Contact Configuration +LOGS_LINK= #URL to logs for api, included in error email. Example: https://console.cloud.google.com/run/detail/us-central1/rsu-manager-cloud-run-api/logs?authuser=1&project=cdot-oim-cv-dev +ENVIRONMENT_NAME= #Environment name, just to display in email. Example: cdot-oim-cv-dev + +# Moove AI feature environment variables +GOOGLE_ACCESS_KEY_NAME=sample_gcp_service_account.json +GCP_PROJECT_ID= +MOOVE_AI_SEGMENT_AGG_STATS_TABLE= +MOOVE_AI_SEGMENT_EVENT_STATS_TABLE= + +######## -------- "webapp" Docker Profile Services +# Run webapp service for cvmanager +# Requires: basic +# Compose file: docker-compose.yml +# cvmanager_webapp +# - React frontend for cvmanager + +# Mapbox token for map rendering in the webapp +MAPBOX_TOKEN= + +# DOT_NAME must be set for the DOT name to correctly populate when building an image for deployment +DOT_NAME="CDOT" + +# Initial map viewport +MAPBOX_INIT_LATITUDE="39.7392" +MAPBOX_INIT_LONGITUDE="-104.9903" +MAPBOX_INIT_ZOOM="10" + +VIEWER_MSG_TYPES='BSM' + +# Both MUST not have trailing slashes +CVIZ_API_SERVER_URL=http://${DOCKER_HOST_IP}:8089 +CVIZ_API_WS_URL=ws://${DOCKER_HOST_IP}:8089 + +# Webapp themes: dark +# base theme is used by default, dark theme is used if browser is set to dark mode +WEBAPP_THEME_LIGHT="dark" # if not set, defaults to 'dark' +WEBAPP_THEME_DARK="dark" # if not set, defaults to 'dark' + +# Webapp logo to use, imported into docker image as volume. Set the full path to the image, for light and dark mode +WEBAPP_LOGO_PNG_ROOT_FILE_PATH_LIGHT=./webapp/cdot_icon.png +WEBAPP_LOGO_PNG_ROOT_FILE_PATH_DARK=./webapp/cdot_icon.png + + +######## -------- "intersection" Docker Profile Services +# Run connected intersection services +# Requires: basic +# Compose file: docker-compose-intersection.yml +# Services: +# - kafka +# - Message broker for communication between conflictmonitor intersection services +# - kafka_init +# - Initialize kafka topics, then die +# - intersection_api +# - Java backend api for intersection/conflictmonitor services +# - mongodb_container +# - MongoDB database for intersection/conflictmonitor data + +#### ---- intersection_api + +# GitHub Token (Required for Intersection API) - See services/intersection-api/README.md#github-token for steps to generate +MAVEN_GITHUB_TOKEN= +MAVEN_GITHUB_ORG=usdot-jpo-ode + +#Specify MongoDB connection parameters +DB_HOST_IP=${DOCKER_HOST_IP} +DB_HOST_PORT=27017 + +KAFKA_BOOTSTRAP_SERVERS=${DOCKER_HOST_IP}:9092 +KAFKA_BROKER_PORT=9092 + +CM_MAXIMUM_RESPONSE_SIZE=10000 + +CM_SERVER_URL=http://${DOCKER_HOST_IP}:8082 +KAFKA_BROKER_IP=${DOCKER_HOST_IP} + +# Startup delay of intersection_api, to wait for kafka topics to be created by kafka_init +CM_STARTUP_DELAY_SECONDS=90 + +# Enable or Disable Features of the Intersection API, for rest endpoints, notification emailer task, and report generation task +INTERSECTION_API_ENABLE_API=true +INTERSECTION_API_ENABLE_EMAILER=true +INTERSECTION_API_ENABLE_REPORTS=true +INTERSECTION_API_ENABLE_HAAS=true + +# Base Path Prefix - Insert the following path into the base route of all REST endpoints in the Intersection API, used for simplifying proxy routing +# Routes MUST start with a slash +INTERSECTION_API_ROUTE_PREFIX="/" + +# Email Configuration +INTERSECTION_EMAIL_BROKER="" # sendgrid, postmark, anything else will use generic SMTP mail server + +# if EMAIL_BROKER is not set (SMTP): +INTERSECTION_SENDER_EMAIL= +INTERSECTION_SMTP_SERVER_IP= +INTERSECTION_SMTP_SERVER_PORT=1025 + +# if EMAIL_BROKER="sendgrid": +SENDGRID_USERNAME= +SENDGRID_PASSWORD= + +# if EMAIL_BROKER="postmark": +POSTMARK_SERVER_TOKEN= + +#### ---- mongodb_container + +# The username and password to use for accessing mongoDB. +MONGO_READ_WRITE_USER=ode +MONGO_READ_WRITE_PASS=replace_me + +# Generate a random string for the MongoDB keyfile using the following command: +# $ openssl rand -base64 32 +MONGO_DB_KEYFILE_STRING=replacethisstring + +CM_DATABASE_NAME=CV +MONGO_DATA_RETENTION_SECONDS=31536000 # 1 year +MONGO_DATABASE_MAX_TTL_RETENTION_SECONDS=31536000 # 1 year +# if set to nothing, the import will be skipped +MONGO_SAMPLE_DATA_RELATIVE_PATH=../resources/mongodumps/dump_2025_07_21 + +MONGO_DB_URI="mongodb://${MONGO_READ_WRITE_USER}:${MONGO_READ_WRITE_PASS}@${DB_HOST_IP}:${DB_HOST_PORT}/?directConnection=true&authSource=admin" +MONGO_DB_NAME=${CM_DATABASE_NAME} + + +######## -------- "ADM" Docker Profile Services +ADM_LOG_TO_FILE=false +ADM_LOG_TO_CONSOLE=true +ADM_LOG_LEVEL=INFO + + +######## -------- "AEM" Docker Profile Services +AEM_LOG_TO_FILE=false +AEM_LOG_TO_CONSOLE=true +AEM_LOG_LEVEL=INFO + + +######## -------- "intersection_no_api" Docker Profile Services +# Run connected intersection services without intersection_api +# Requires: basic +# Compose file: docker-compose-intersection.yml +# Services: +# - kafka +# - Message broker for communication between conflictmonitor intersection services +# - kafka_init +# - Initialize kafka topics, then die +# - mongodb_container +# - MongoDB database for intersection/conflictmonitor data + +# No additional variables - see intersection variables + +######## -------- "conflictmonitor" Docker Profile Services +# Run connected conflictmonitor services +# Requires: basic, intersection | intersection_no_api +# Compose file: docker-compose-conflictmonitor.yml +# Services: +# - conflictmonitor +# - Java-based kafka streaming service, generates events, assessments, and notifications from intersection data +# - ode +# - Java-based kafka streaming service, processes raw J2735 messages +# - geojsonconverter +# - Java-based kafka streaming service, generates enhanced geojson-based messages +# - connect +# - Kafka connect service, backs up data on kafka topics to MongoDB + +#### ---- conflictmonitor +RESTART_POLICY="on-failure:3" + +# RocksDB Bounded Memory Config Properties +# 128 MB = 134217728 +# 64 MB = 67108864 +# 16 MB = 16777216 +ROCKSDB_TOTAL_OFF_HEAP_MEMORY=134217728 +ROCKSDB_INDEX_FILTER_BLOCK_RATIO=0.1 +ROCKSDB_TOTAL_MEMTABLE_MEMORY=67108864 +ROCKSDB_BLOCK_SIZE=4096 +ROCKSDB_N_MEMTABLES=2 +ROCKSDB_MEMTABLE_SIZE=16777216 + +######## -------- "deduplicator" Docker Profile services +KAFKA_TOPIC_CREATE_DEDUPLICATOR=true +CONNECT_CREATE_DEDUPLICATOR=true + +#### ---- connect +CONNECT_URL=http://${DOCKER_HOST_IP}:8083 + +######## -------- "addons" Docker Profile Services +# Run all cvmanager helper microservices +# Requires: None +# Compose file: docker-compose-addons.yml +# Services: jpo_count_metric, rsu_status_check, jpo_iss_health_check, firmware_manager_upgrade_scheduler, firmware_manager_upgrade_runner +# - jpo_count_metric +# - Generates counts emails for various data types ("BSM", "TIM", "Map", "SPaT", "SRM", "SSM"). Can store in MongoDB or BigQuery +# - rsu_status_check +# - Checks status of RSUs and stores in Postgres +# - jpo_iss_health_check +# - Retrieves ISS health into and stores in Postgres +# - firmware_manager_upgrade_scheduler +# - Compares RSU firmware versions with Postgres and schedules firmware_manager_upgrade_runner +# - firmware_manager_upgrade_runner +# - Completes RSU firmware upgrades + +#### ---- jpo_count_metric + +# Count Metric Addon: +ENABLE_EMAILER='True' + +# If ENABLE_EMAILER is 'True', set the following environment variables +DEPLOYMENT_TITLE='JPO-ODE' + +# SMTP REQUIRED VARIABLES +SMTP_SERVER_IP='' +SMTP_USERNAME='' +SMTP_PASSWORD='' +SMTP_EMAIL='' + +# If ENABLE_EMAILER is 'False', set the following environment variables +ODE_KAFKA_BROKERS=${DOCKER_HOST_IP}:9092 + +# EITHER "MONGODB" or "BIGQUERY" +COUNT_DESTINATION_DB='MONGODB' + +# MONGODB REQUIRED VARIABLES +INPUT_COUNTS_MONGO_COLLECTION_NAME='' +OUTPUT_COUNTS_MONGO_COLLECTION_NAME='' + +KAFKA_BIGQUERY_TABLENAME= + +#### ---- rsu_status_check + +# Services that can be toggled on or off +# 'True' or 'False' are the only legal values + +# Toggles monitoring of RSU online status +RSU_PING=True + +# Fetches ping data from Zabbix - alternatively the service will ping the RSUs on its own +# Only used when RSU_PING is 'True' +ZABBIX=False + +# Fetches SNMP configuration data for all RSUs +RSU_MSGFWD_FETCH=True +RSU_SECURITY_FETCH=True +RSU_HEALTH_FETCH=True + +# Zabbix endpoint and API authentication +# Only used when ZABBIX is 'True' +ZABBIX_ENDPOINT= +ZABBIX_USER= +ZABBIX_PASSWORD= + +# Customize the period at which the purger will determine a ping log is too old and will be deleted +# Number of hours +STALE_PERIOD=24 + +#### ---- jpo_iss_health_check + +# Key Storage +## Type of key storage, options: gcp, postgres +STORAGE_TYPE=postgres + +# ISS Account Authentication +ISS_API_KEY= +ISS_API_KEY_NAME= +ISS_PROJECT_ID= +ISS_SCMS_TOKEN_REST_ENDPOINT= +ISS_SCMS_VEHICLE_REST_ENDPOINT= + +## Postgres Storage (Required if STORAGE_TYPE=postgres) +### Table name to store keys +ISS_KEY_TABLE_NAME= + +#### ---- firmware_manager_upgrade_runner + +BLOB_STORAGE_PROVIDER=DOCKER +BLOB_STORAGE_BUCKET= + +## Docker volume mount point for BLOB storage (if using Docker) +HOST_BLOB_STORAGE_DIRECTORY=./local_blob_storage + +## Maximum retry limit for performing firmware upgrades +FW_UPGRADE_MAX_RETRY_LIMIT=3 + +FIRMWARE_MANAGER_UPGRADE_SCHEDULER_ENDPOINT=http://${DOCKER_HOST_IP}:8089 + +#### ---- firmware_manager_upgrade_scheduler + +FIRMWARE_MANAGER_UPGRADE_RUNNER_ENDPOINT=http://${DOCKER_HOST_IP}:8090 + +######## -------- "obu_ota" Docker Profile Services +# Run OBU over-the-air update microservices +# Requires: None +# Compose file: docker-compose-obu-ota-server.yml +# Services: jpo_ota_backend, jpo_ota_nginx +# - jpo_ota_backend +# - Over-the-air update microservice for OBUs +# - jpo_ota_nginx +# - NGINX proxy for OBU OTA backend + + +#### ---- jpo_ota_backend + +# Route-able hostname for the server +OBU_OTA_SERVER_HOST={DOCKER_HOST_IP} + +# For users using GCP cloud storage +OBU_OTA_BLOB_STORAGE_BUCKET= +OBU_OTA_BLOB_STORAGE_PATH= + +# Nginx basic auth username and password +OTA_USERNAME="admin" +OTA_PASSWORD="admin" + +# Max number of successful firmware upgrades to keep in the database per device SN +MAX_COUNT=10 + +# Nginx encryption options: "plain", "ssl" +# Note that this just changes the config file attached as a volume to the Nginx container +NGINX_ENCRYPTION="plain" + +#### ---- jpo_ota_nginx + +# SSL file name in path /docker/nginx/ssl/ +SERVER_CERT_FILE="ota_server.crt" +SERVER_KEY_FILE="ota_server.key" diff --git a/sample.env b/sample.env index 1361ca207..029be5773 100644 --- a/sample.env +++ b/sample.env @@ -5,453 +5,27 @@ # Additionally, each individual service in this project can be started by specifying its service name as a profile. # The currently available profile groups are listed below. # basic, webapp, mongo_full, kafka_full, intersection, intersection_no_api, conflictmonitor, addons, obu_ota, kafka_connect_standalone -COMPOSE_PROFILES=basic,webapp,mongo_full,kafka_full,intersection,kafka_connect_standalone +COMPOSE_PROFILES=basic,webapp,intersection,mongo_full,kafka_full,kafka_connect_standalone,conflictmonitor -######## -------- General Variables - Apply to All Profiles +######## -------- Required Variables +# run 'ifconfig' in wsl DOCKER_HOST_IP= -KEYCLOAK_DOMAIN=cvmanager.auth.com -KC_HOST_IP=${DOCKER_HOST_IP} -KEYCLOAK_ENDPOINT=http://${KEYCLOAK_DOMAIN}:8084 - -WEBAPP_DOMAIN=cvmanager.local.com -WEBAPP_HOST_IP=${DOCKER_HOST_IP} -WEBAPP_ENDPOINT=http://${WEBAPP_DOMAIN} - -API_ENDPOINT=http://cvmanager.local.com:8081 - -# Remote dockerhub variables for ODE and ConflictMonitor images -DOCKERHUB_HOST=usdotjpoode -DOCKERHUB_RELEASE=2025-q2 - -# Logging Levels - "DEBUG", "INFO", "WARNING", "ERROR" -API_LOGGING_LEVEL="INFO" -FIRMWARE_MANAGER_LOGGING_LEVEL="INFO" -GEO_LOGGING_LEVEL="INFO" -ISS_LOGGING_LEVEL="INFO" -RSU_STATUS_LOGGING_LEVEL="INFO" -COUNTS_LOGGING_LEVEL="INFO" -OBU_OTA_LOGGING_LEVEL="INFO" -KC_LOGGING_LEVEL="INFO" # Also includes "ALL", "FATAL", "OFF", "TRACE" and "WARN" - -# Feature Flags -ENABLE_RSU_FEATURES='true' # 'false' to disable -ENABLE_INTERSECTION_FEATURES='true' # 'false' to disable -ENABLE_WZDX_FEATURES='true' # 'false' to disable -ENABLE_MOOVE_AI_FEATURES='true' # 'false' to disable -ENABLE_HAAS_FEATURES='true' # 'false' to disable - - -######## -------- "basic" Docker Profile Services -# Run critical cvmanager services -# Requires: None -# Compose file: docker-compose.yml -# Services: -# - cvmanager_api -# - Python backend api for webapp -# - cvmanager_postgres -# - Postgres database for cvmanager data and backing database for keycloak instance -# - cvmanager_keycloak -# - Keycloak instance for user authentication and authorization of webapp and api requests - -#### ---- cvmanager_keycloak -# Keycloak authentication credentials -KEYCLOAK_ADMIN=admin -KEYCLOAK_ADMIN_PASSWORD=admin - -# Keycloak Parameters - to generate secret key use a password generator such as: https://www.avast.com/en-us/random-password-generator#pc and set the length to 32 -KEYCLOAK_REALM=cvmanager -KEYCLOAK_GUI_CLIENT_ID=cvmanager-gui -KEYCLOAK_API_CLIENT_ID=cvmanager-api -KEYCLOAK_API_CLIENT_SECRET_KEY=w8zpoArUwIVN6TSDY5WQgX9TlVAgH9OF -KEYCLOAK_LOGIN_THEME_NAME=sample_theme - -# GCP OAuth2.0 client ID for SSO authentication in keycloak - if not specified the google SSO will not be functional -GOOGLE_CLIENT_ID= -GOOGLE_CLIENT_SECRET= - -#### ---- cvmanager_postgres - -# PostgreSQL Database connection information -# this value may need to follow with the webapp host if debugging the applications -PG_DB_HOST=${DOCKER_HOST_IP}:5432 -PG_DB_NAME=postgres -PG_DB_USER=postgres -# If the PG_DB_PASS variable has special characters, make sure to wrap it in single quotes -PG_DB_PASS=postgres - -#### ---- cvmanager_api - -# Allowed CORS domain for accessing the CV Manager API from (set to the web application hostname) -# Make sure to include http:// or https:// -# If using docker then this value should be set to: http://${WEBAPP_HOST_IP}:3000 -# If running the webapp using npm then set it to: http://localhost:3000 -# Leave as * to allow all domains access -CORS_DOMAIN=* - -# Set these variables if using either "MONGODB" or "BIGQUERY" for COUNT_DESTINATION_DB of jpo_count_metric -# COUNTS_MSG_TYPES: Comma separated list of message types. -# COUNTS_MSG_TYPES must be set for the counts menu to correctly populate when building an image for deployment -COUNTS_MSG_TYPES='BSM,SSM,SPAT,SRM,MAP' -MONGO_PROCESSED_BSM_COLLECTION_NAME='ProcessedBsm' -MONGO_PROCESSED_PSM_COLLECTION_NAME='ProcessedPsm' -SSM_DB_NAME= -SRM_DB_NAME= - -# Specifies the maximum number of V2X messages returned from the geo_query_geo_data_mongo method before filtering occurs -MAX_GEO_QUERY_RECORDS= - -# If running firmware manager addon -FIRMWARE_MANAGER_ENDPOINT=http://${DOCKER_HOST_IP}:8089 - -# If connecting to PGDB over websocket: -INSTANCE_CONNECTION_NAME= - -# Python timezone for the CV Manager (You can list pytz timezones with the command 'pytz.all_timezones') -TIMEZONE="US/Mountain" - -# WZDx API key and endpoint for pulling WZDx data into the CV Manager -WZDX_API_KEY= -WZDX_ENDPOINT=data.cotrip.org - -# Contact Support Menu Email Configuration -CSM_EMAIL_TO_SEND_FROM= -CSM_EMAIL_APP_USERNAME= -CSM_EMAIL_APP_PASSWORD= -CSM_EMAILS_TO_SEND_TO= -CSM_TARGET_SMTP_SERVER_ADDRESS= -CSM_TARGET_SMTP_SERVER_PORT=587 -CSM_TLS_ENABLED=true -CSM_AUTH_ENABLED=true - -# Error Email Contact Configuration -LOGS_LINK= #URL to logs for api, included in error email. Example: https://console.cloud.google.com/run/detail/us-central1/rsu-manager-cloud-run-api/logs?authuser=1&project=cdot-oim-cv-dev -ENVIRONMENT_NAME= #Environment name, just to display in email. Example: cdot-oim-cv-dev - -# Moove AI feature environment variables -GOOGLE_ACCESS_KEY_NAME=sample_gcp_service_account.json -GCP_PROJECT_ID= -MOOVE_AI_SEGMENT_AGG_STATS_TABLE= -MOOVE_AI_SEGMENT_EVENT_STATS_TABLE= - -######## -------- "webapp" Docker Profile Services -# Run webapp service for cvmanager -# Requires: basic -# Compose file: docker-compose.yml -# cvmanager_webapp -# - React frontend for cvmanager - # Mapbox token for map rendering in the webapp MAPBOX_TOKEN= -# DOT_NAME must be set for the DOT name to correctly populate when building an image for deployment -DOT_NAME="CDOT" - -# Initial map viewport -MAPBOX_INIT_LATITUDE="39.7392" -MAPBOX_INIT_LONGITUDE="-104.9903" -MAPBOX_INIT_ZOOM="10" - -VIEWER_MSG_TYPES='BSM' - -# Both MUST not have trailing slashes -CVIZ_API_SERVER_URL=http://${DOCKER_HOST_IP}:8089 -CVIZ_API_WS_URL=ws://${DOCKER_HOST_IP}:8089 - -# Webapp themes: dark -# base theme is used by default, dark theme is used if browser is set to dark mode -WEBAPP_THEME_LIGHT="dark" # if not set, defaults to 'dark' -WEBAPP_THEME_DARK="dark" # if not set, defaults to 'dark' - -# Webapp logo to use, imported into docker image as volume. Set the full path to the image, for light and dark mode -WEBAPP_LOGO_PNG_ROOT_FILE_PATH_LIGHT=./webapp/cdot_icon.png -WEBAPP_LOGO_PNG_ROOT_FILE_PATH_DARK=./webapp/cdot_icon.png - - -######## -------- "intersection" Docker Profile Services -# Run connected intersection services -# Requires: basic -# Compose file: docker-compose-intersection.yml -# Services: -# - kafka -# - Message broker for communication between conflictmonitor intersection services -# - kafka_init -# - Initialize kafka topics, then die -# - intersection_api -# - Java backend api for intersection/conflictmonitor services -# - mongodb_container -# - MongoDB database for intersection/conflictmonitor data - -#### ---- intersection_api - # GitHub Token (Required for Intersection API) - See services/intersection-api/README.md#github-token for steps to generate MAVEN_GITHUB_TOKEN= -MAVEN_GITHUB_ORG=usdot-jpo-ode - -#Specify MongoDB connection parameters -DB_HOST_IP=${DOCKER_HOST_IP} -DB_HOST_PORT=27017 - -KAFKA_BOOTSTRAP_SERVERS=${DOCKER_HOST_IP}:9092 -KAFKA_BROKER_PORT=9092 - -CM_MAXIMUM_RESPONSE_SIZE=10000 - -CM_SERVER_URL=http://${DOCKER_HOST_IP}:8082 -KAFKA_BROKER_IP=${DOCKER_HOST_IP} - -# Startup delay of intersection_api, to wait for kafka topics to be created by kafka_init -CM_STARTUP_DELAY_SECONDS=90 - -# Enable or Disable Features of the Intersection API, for rest endpoints, notification emailer task, and report generation task -INTERSECTION_API_ENABLE_API=true -INTERSECTION_API_ENABLE_EMAILER=true -INTERSECTION_API_ENABLE_REPORTS=true -INTERSECTION_API_ENABLE_HAAS=true - -# Base Path Prefix - Insert the following path into the base route of all REST endpoints in the Intersection API, used for simplifying proxy routing -# Routes MUST start with a slash -INTERSECTION_API_ROUTE_PREFIX="/" - -# ASN.1 FFM Decoder Config -J2735_ASN_TEXT_BUFFER_SIZE=262144 -J2735_UPER_BUFFER_SIZE=8192 -J2735_ERROR_BUFFER_SIZE=256 - -# Email Configuration -INTERSECTION_EMAIL_BROKER="" # sendgrid, postmark, anything else will use generic SMTP mail server - -# if EMAIL_BROKER is not set (SMTP): -INTERSECTION_SENDER_EMAIL= -INTERSECTION_SMTP_SERVER_IP= -INTERSECTION_SMTP_SERVER_PORT=1025 - -# if EMAIL_BROKER="sendgrid": -SENDGRID_USERNAME= -SENDGRID_PASSWORD= - -# if EMAIL_BROKER="postmark": -POSTMARK_SERVER_TOKEN= - -#### ---- mongodb_container - -# The username and password to use for accessing mongoDB. -MONGO_READ_WRITE_USER=ode -MONGO_READ_WRITE_PASS=replace_me -# Generate a random string for the MongoDB keyfile using the following command: -# $ openssl rand -base64 32 -MONGO_DB_KEYFILE_STRING=replacethisstring -CM_DATABASE_NAME=CV -MONGO_DATA_RETENTION_SECONDS=31536000 # 1 year -MONGO_DATABASE_MAX_TTL_RETENTION_SECONDS=31536000 # 1 year -# if set to nothing, the import will be skipped -MONGO_SAMPLE_DATA_RELATIVE_PATH=../resources/mongodumps/dump_2025_07_21 +######## -------- Optional (but recommended) Variables -MONGO_DB_URI="mongodb://${MONGO_READ_WRITE_USER}:${MONGO_READ_WRITE_PASS}@${DB_HOST_IP}:${DB_HOST_PORT}/?directConnection=true&authSource=admin" -MONGO_DB_NAME=${CM_DATABASE_NAME} +# Load sample intersection data +MONGO_SAMPLE_DATA_RELATIVE_PATH=../resources/mongodumps/dump_2025_10_15 - -######## -------- "ADM" Docker Profile Services -ADM_LOG_TO_FILE=false -ADM_LOG_TO_CONSOLE=true -ADM_LOG_LEVEL=INFO - - -######## -------- "AEM" Docker Profile Services -AEM_LOG_TO_FILE=false -AEM_LOG_TO_CONSOLE=true -AEM_LOG_LEVEL=INFO - - -######## -------- "intersection_no_api" Docker Profile Services -# Run connected intersection services without intersection_api -# Requires: basic -# Compose file: docker-compose-intersection.yml -# Services: -# - kafka -# - Message broker for communication between conflictmonitor intersection services -# - kafka_init -# - Initialize kafka topics, then die -# - mongodb_container -# - MongoDB database for intersection/conflictmonitor data - -# No additional variables - see intersection variables - -######## -------- "conflictmonitor" Docker Profile Services -# Run connected conflictmonitor services -# Requires: basic, intersection | intersection_no_api -# Compose file: docker-compose-conflictmonitor.yml -# Services: -# - conflictmonitor -# - Java-based kafka streaming service, generates events, assessments, and notifications from intersection data -# - ode -# - Java-based kafka streaming service, processes raw J2735 messages -# - geojsonconverter -# - Java-based kafka streaming service, generates enhanced geojson-based messages -# - connect -# - Kafka connect service, backs up data on kafka topics to MongoDB - -#### ---- conflictmonitor -RESTART_POLICY="on-failure:3" - -# RocksDB Bounded Memory Config Properties -# 128 MB = 134217728 -# 64 MB = 67108864 -# 16 MB = 16777216 -ROCKSDB_TOTAL_OFF_HEAP_MEMORY=134217728 -ROCKSDB_INDEX_FILTER_BLOCK_RATIO=0.1 -ROCKSDB_TOTAL_MEMTABLE_MEMORY=67108864 -ROCKSDB_BLOCK_SIZE=4096 -ROCKSDB_N_MEMTABLES=2 -ROCKSDB_MEMTABLE_SIZE=16777216 - -######## -------- "deduplicator" Docker Profile services -KAFKA_TOPIC_CREATE_DEDUPLICATOR=true -CONNECT_CREATE_DEDUPLICATOR=true - -#### ---- connect -CONNECT_URL=http://${DOCKER_HOST_IP}:8083 - -######## -------- "addons" Docker Profile Services -# Run all cvmanager helper microservices -# Requires: None -# Compose file: docker-compose-addons.yml -# Services: jpo_count_metric, rsu_status_check, jpo_iss_health_check, firmware_manager_upgrade_scheduler, firmware_manager_upgrade_runner -# - jpo_count_metric -# - Generates counts emails for various data types ("BSM", "TIM", "Map", "SPaT", "SRM", "SSM"). Can store in MongoDB or BigQuery -# - rsu_status_check -# - Checks status of RSUs and stores in Postgres -# - jpo_iss_health_check -# - Retrieves ISS health into and stores in Postgres -# - firmware_manager_upgrade_scheduler -# - Compares RSU firmware versions with Postgres and schedules firmware_manager_upgrade_runner -# - firmware_manager_upgrade_runner -# - Completes RSU firmware upgrades - -#### ---- jpo_count_metric - -# Count Metric Addon: -ENABLE_EMAILER='True' - -# If ENABLE_EMAILER is 'True', set the following environment variables -DEPLOYMENT_TITLE='JPO-ODE' - -# SMTP REQUIRED VARIABLES -SMTP_SERVER_IP='' -SMTP_USERNAME='' -SMTP_PASSWORD='' -SMTP_EMAIL='' - -# If ENABLE_EMAILER is 'False', set the following environment variables -COUNT_MESSAGE_TYPES='bsm' -ODE_KAFKA_BROKERS=${DOCKER_HOST_IP}:9092 - -# EITHER "MONGODB" or "BIGQUERY" -COUNT_DESTINATION_DB='MONGODB' - -# MONGODB REQUIRED VARIABLES -INPUT_COUNTS_MONGO_COLLECTION_NAME='' -OUTPUT_COUNTS_MONGO_COLLECTION_NAME='' - -KAFKA_BIGQUERY_TABLENAME= - -#### ---- rsu_status_check - -# Services that can be toggled on or off -# 'True' or 'False' are the only legal values - -# Toggles monitoring of RSU online status -RSU_PING=True - -# Fetches ping data from Zabbix - alternatively the service will ping the RSUs on its own -# Only used when RSU_PING is 'True' -ZABBIX=False - -# Fetches SNMP configuration data for all RSUs -RSU_MSGFWD_FETCH=True -RSU_SECURITY_FETCH=True -RSU_HEALTH_FETCH=True - -# Zabbix endpoint and API authentication -# Only used when ZABBIX is 'True' -ZABBIX_ENDPOINT= -ZABBIX_USER= -ZABBIX_PASSWORD= - -# Customize the period at which the purger will determine a ping log is too old and will be deleted -# Number of hours -STALE_PERIOD=24 - -#### ---- jpo_iss_health_check - -# Key Storage -## Type of key storage, options: gcp, postgres -STORAGE_TYPE=postgres - -# ISS Account Authentication -ISS_API_KEY= -ISS_API_KEY_NAME= -ISS_PROJECT_ID= -ISS_SCMS_TOKEN_REST_ENDPOINT= -ISS_SCMS_VEHICLE_REST_ENDPOINT= - -## Postgres Storage (Required if STORAGE_TYPE=postgres) -### Table name to store keys -ISS_KEY_TABLE_NAME= - -#### ---- firmware_manager_upgrade_runner - -BLOB_STORAGE_PROVIDER=DOCKER -BLOB_STORAGE_BUCKET= - -## Docker volume mount point for BLOB storage (if using Docker) -HOST_BLOB_STORAGE_DIRECTORY=./local_blob_storage - -## Maximum retry limit for performing firmware upgrades -FW_UPGRADE_MAX_RETRY_LIMIT=3 - -FIRMWARE_MANAGER_UPGRADE_SCHEDULER_ENDPOINT=http://${DOCKER_HOST_IP}:8089 - -#### ---- firmware_manager_upgrade_scheduler - -FIRMWARE_MANAGER_UPGRADE_RUNNER_ENDPOINT=http://${DOCKER_HOST_IP}:8090 - -######## -------- "obu_ota" Docker Profile Services -# Run OBU over-the-air update microservices -# Requires: None -# Compose file: docker-compose-obu-ota-server.yml -# Services: jpo_ota_backend, jpo_ota_nginx -# - jpo_ota_backend -# - Over-the-air update microservice for OBUs -# - jpo_ota_nginx -# - NGINX proxy for OBU OTA backend - - -#### ---- jpo_ota_backend - -# Route-able hostname for the server -OBU_OTA_SERVER_HOST={DOCKER_HOST_IP} - -# For users using GCP cloud storage -OBU_OTA_BLOB_STORAGE_BUCKET= -OBU_OTA_BLOB_STORAGE_PATH= - -# Nginx basic auth username and password -OTA_USERNAME="admin" -OTA_PASSWORD="admin" - -# Max number of successful firmware upgrades to keep in the database per device SN -MAX_COUNT=10 - -# Nginx encryption options: "plain", "ssl" -# Note that this just changes the config file attached as a volume to the Nginx container -NGINX_ENCRYPTION="plain" - -#### ---- jpo_ota_nginx - -# SSL file name in path /docker/nginx/ssl/ -SERVER_CERT_FILE="ota_server.crt" -SERVER_KEY_FILE="ota_server.key" +# WZDx API key and endpoint for pulling WZDx data into the CV Manager (See https://maps.cotrip.org/help/117/Traveler-Information-Data-Feed-Access to generate an API key) +# Make sure to uncomment the next line if setting the WZDx api key +# ENABLE_WZDX_FEATURES='true' +WZDX_API_KEY= +WZDX_ENDPOINT=data.cotrip.org \ No newline at end of file diff --git a/services/addons/images/count_metric/count_metric_environment.py b/services/addons/images/count_metric/count_metric_environment.py new file mode 100644 index 000000000..8d6051c09 --- /dev/null +++ b/services/addons/images/count_metric/count_metric_environment.py @@ -0,0 +1,10 @@ +from common.common_environment import get_env_var + +SMTP_SERVER_IP=get_env_var("SMTP_SERVER_IP", error=True) +SMTP_SERVER_PORT=int(get_env_var("SMTP_SERVER_PORT", "587", warn=False)) +SMTP_EMAIL=get_env_var("SMTP_EMAIL", error=True) +DEPLOYMENT_TITLE=get_env_var("DEPLOYMENT_TITLE", "Example Deployment", warn=True) +SMTP_USERNAME=get_env_var("SMTP_USERNAME", error=True) +SMTP_PASSWORD=get_env_var("SMTP_PASSWORD", error=True) +MONGO_DB_URI=get_env_var("MONGO_DB_URI", "mongodb://localhost:27017") +MONGO_DB_NAME=get_env_var("MONGO_DB_NAME", "CV") diff --git a/services/addons/images/count_metric/daily_emailer.py b/services/addons/images/count_metric/daily_emailer.py index 86e98e83c..62b382d66 100644 --- a/services/addons/images/count_metric/daily_emailer.py +++ b/services/addons/images/count_metric/daily_emailer.py @@ -1,4 +1,3 @@ -import os import logging import gen_email from common.emailSender import EmailSender @@ -6,6 +5,7 @@ from common.email_util import get_email_list from datetime import datetime, timedelta from pymongo import MongoClient +import count_metric_environment message_types = ["BSM", "TIM", "Map", "SPaT", "SRM", "SSM"] @@ -129,17 +129,17 @@ def email_daily_counts(org_name, email_body): for email_address in email_addresses: emailSender = EmailSender( - os.environ["SMTP_SERVER_IP"], - 587, + count_metric_environment.SMTP_SERVER_IP, + count_metric_environment.SMTP_SERVER_PORT, ) emailSender.send( - sender=os.environ["SMTP_EMAIL"], + sender=count_metric_environment.SMTP_EMAIL, recipient=email_address, - subject=f"{org_name} {str(os.environ['DEPLOYMENT_TITLE'])} Counts", + subject=f"{org_name} {count_metric_environment.DEPLOYMENT_TITLE} Counts", message=email_body, replyEmail="", - username=os.environ["SMTP_USERNAME"], - password=os.environ["SMTP_PASSWORD"], + username=count_metric_environment.SMTP_USERNAME, + password=count_metric_environment.SMTP_PASSWORD, pretty=True, ) except Exception as e: @@ -147,8 +147,8 @@ def email_daily_counts(org_name, email_body): def run_daily_emailer(): - client = MongoClient(os.getenv("MONGO_DB_URI")) - mongo_db = client[os.getenv("MONGO_DB_NAME")] + client = MongoClient(count_metric_environment.MONGO_DB_URI) + mongo_db = client[count_metric_environment.MONGO_DB_NAME] # Grab today's date and yesterday's date for a 24 hour range start_dt = (datetime.now() - timedelta(1)).replace( diff --git a/services/addons/images/count_metric/gen_email.py b/services/addons/images/count_metric/gen_email.py index b0da6058b..fb56fb0a8 100644 --- a/services/addons/images/count_metric/gen_email.py +++ b/services/addons/images/count_metric/gen_email.py @@ -1,6 +1,6 @@ import logging -import os from datetime import datetime +import count_metric_environment def diff_to_color(val): @@ -39,7 +39,7 @@ def generate_table_row(rsu_ip, data, row_style, message_type_list): def generate_count_table(rsu_dict, message_type_list): - logging.info(f"Creating count table...") + logging.info("Creating count table...") # If the RSU dictionary is completely empty, return nothing to indicate an issue has occurred somewhere if not rsu_dict: @@ -97,7 +97,7 @@ def generate_email_body(org_name, rsu_dict, start_dt, end_dt, message_type_list) # DEPLOYMENT_TITLE is a contextual title for where these counts apply. ie. "GCP prod" # This is generalized to support any deployment environment html = ( - f'

{org_name} {str(os.environ["DEPLOYMENT_TITLE"])} Count Report {start} UTC - {end} UTC

' + f"

{org_name} {count_metric_environment.DEPLOYMENT_TITLE} Count Report {start} UTC - {end} UTC

" "

This is an automated email to report yesterday's ODE message counts for J2735 messages going in and out of the ODE. " "In counts are the number of encoded messages received by the ODE from the load balancer. " "Out counts are the number of decoded messages that have come out of the ODE in JSON form and " diff --git a/services/addons/images/count_metric/mongo_counter.py b/services/addons/images/count_metric/mongo_counter.py index 1f99711ea..c05b981ae 100644 --- a/services/addons/images/count_metric/mongo_counter.py +++ b/services/addons/images/count_metric/mongo_counter.py @@ -1,7 +1,7 @@ -import os import logging from pymongo import MongoClient from datetime import datetime, timedelta +import count_metric_environment message_types = ["BSM", "TIM", "Map", "SPaT", "SRM", "SSM", "PSM"] @@ -79,7 +79,7 @@ def run_mongo_counter(mongo_db): if __name__ == "__main__": logging.info("Starting the MongoDB counter") - client = MongoClient(os.getenv("MONGO_DB_URI")) - mongo_db = client[os.getenv("MONGO_DB_NAME")] + client = MongoClient(count_metric_environment.MONGO_DB_URI) + mongo_db = client[count_metric_environment.MONGO_DB_NAME] run_mongo_counter(mongo_db) logging.info("MongoDB counter has finished") diff --git a/services/addons/images/count_metric/sample.env b/services/addons/images/count_metric/sample.env index 4927c0ff9..0e5f7d94d 100644 --- a/services/addons/images/count_metric/sample.env +++ b/services/addons/images/count_metric/sample.env @@ -25,7 +25,6 @@ SMTP_EMAIL = '' # --------------------------------------------------------------------- # If ENABLE_EMAILER is 'False', set the following environment variables -MESSAGE_TYPES = 'bsm' PROJECT_ID = '' ODE_KAFKA_BROKERS = ':9092' diff --git a/services/addons/images/firmware_manager/upgrade_runner/commsignia_upgrader.py b/services/addons/images/firmware_manager/upgrade_runner/commsignia_upgrader.py index b4f13191e..415b14b2e 100644 --- a/services/addons/images/firmware_manager/upgrade_runner/commsignia_upgrader.py +++ b/services/addons/images/firmware_manager/upgrade_runner/commsignia_upgrader.py @@ -4,8 +4,8 @@ import upgrader import json import logging -import os import sys +from common import common_environment class CommsigniaUpgrader(upgrader.UpgraderAbstractClass): @@ -103,8 +103,8 @@ def post_upgrade(self): # Change permissions and execute post upgrade script logging.info("Running post upgrade script for " + self.rsu_ip + "...") - ssh.exec_command(f"chmod +x /tmp/post_upgrade.sh") - _stdin, _stdout, _stderr = ssh.exec_command(f"/tmp/post_upgrade.sh") + ssh.exec_command("chmod +x /tmp/post_upgrade.sh") + _stdin, _stdout, _stderr = ssh.exec_command("/tmp/post_upgrade.sh") decoded_stdout = _stdout.read().decode() logging.info(decoded_stdout) if "ALL OK" not in decoded_stdout: @@ -135,8 +135,10 @@ def post_upgrade(self): # - target_firmware_version # - install_package if __name__ == "__main__": - log_level = os.environ.get("LOGGING_LEVEL", "INFO") - logging.basicConfig(format="%(levelname)s:%(message)s", level=log_level) + logging.info( + "Commsignia upgrader running with LOGGING_LEVEL: " + + str(common_environment.LOGGING_LEVEL) + ) # Trimming outer single quotes from the json.loads upgrade_info = json.loads(sys.argv[1][1:-1]) commsignia_upgrader = CommsigniaUpgrader(upgrade_info) diff --git a/services/addons/images/firmware_manager/upgrade_runner/upgrade_runner.py b/services/addons/images/firmware_manager/upgrade_runner/upgrade_runner.py index 027ffc53d..a15467ef4 100644 --- a/services/addons/images/firmware_manager/upgrade_runner/upgrade_runner.py +++ b/services/addons/images/firmware_manager/upgrade_runner/upgrade_runner.py @@ -4,13 +4,10 @@ from marshmallow import Schema, fields import json import logging -import os +from common import common_environment app = Flask(__name__) -log_level = os.environ.get("LOGGING_LEVEL", "INFO") -logging.basicConfig(format="%(levelname)s:%(message)s", level=log_level) - manufacturer_upgrade_scripts = { "Commsignia": "commsignia_upgrader.py", "Yunex": "yunex_upgrader.py", @@ -96,4 +93,8 @@ def serve_rest_api(): if __name__ == "__main__": + logging.info( + "Firmware manager running with LOGGING_LEVEL: " + + str(common_environment.LOGGING_LEVEL) + ) serve_rest_api() diff --git a/services/addons/images/firmware_manager/upgrade_runner/upgrade_runner_environment.py b/services/addons/images/firmware_manager/upgrade_runner/upgrade_runner_environment.py new file mode 100644 index 000000000..46f464496 --- /dev/null +++ b/services/addons/images/firmware_manager/upgrade_runner/upgrade_runner_environment.py @@ -0,0 +1,11 @@ +from common.common_environment import get_env_var + +BLOB_STORAGE_PROVIDER=get_env_var("BLOB_STORAGE_PROVIDER", "DOCKER", warn=False) +UPGRADE_SCHEDULER_ENDPOINT=get_env_var("UPGRADE_SCHEDULER_ENDPOINT", "127.0.0.1") + +SMTP_SERVER_IP=get_env_var("SMTP_SERVER_IP", error=True) +SMTP_SERVER_PORT=int(get_env_var("SMTP_SERVER_PORT", "587", warn=False)) +SMTP_EMAIL=get_env_var("SMTP_EMAIL", error=True) +DEPLOYMENT_TITLE=get_env_var("DEPLOYMENT_TITLE", "Example Deployment", warn=True) +SMTP_USERNAME=get_env_var("SMTP_USERNAME", error=True) +SMTP_PASSWORD=get_env_var("SMTP_PASSWORD", error=True) diff --git a/services/addons/images/firmware_manager/upgrade_runner/upgrader.py b/services/addons/images/firmware_manager/upgrade_runner/upgrader.py index fb3dd24e5..3398ac2ca 100644 --- a/services/addons/images/firmware_manager/upgrade_runner/upgrader.py +++ b/services/addons/images/firmware_manager/upgrade_runner/upgrader.py @@ -4,12 +4,12 @@ import time from common import gcs_utils import logging -import os import requests import shutil from common.emailSender import EmailSender from common.email_util import get_email_list_from_rsu -import download_blob +from addons.images.firmware_manager.upgrade_runner import download_blob +import upgrade_runner_environment class UpgraderAbstractClass(abc.ABC): @@ -45,9 +45,7 @@ def download_blob( ) # Download blob, defaults to GCP blob storage - bspCaseInsensitive = os.environ.get( - "BLOB_STORAGE_PROVIDER", "DOCKER" - ).casefold() + bspCaseInsensitive = upgrade_runner_environment.BLOB_STORAGE_PROVIDER.casefold() if bspCaseInsensitive == "gcp": return ( gcs_utils.download_gcp_blob( @@ -73,8 +71,8 @@ def notify_firmware_manager(self, success): ) # Obtain the upgrade scheduler endpoint - upgrade_scheduler_endpoint = os.environ.get( - "UPGRADE_SCHEDULER_ENDPOINT", "127.0.0.1" + upgrade_scheduler_endpoint = ( + upgrade_runner_environment.UPGRADE_SCHEDULER_ENDPOINT ) if upgrade_scheduler_endpoint == "UNDEFINED": raise Exception( @@ -132,17 +130,17 @@ def send_error_email(self, type="Firmware Upgrader", err=""): for email_address in email_addresses: emailSender = EmailSender( - os.environ["SMTP_SERVER_IP"], - 587, + upgrade_runner_environment.SMTP_SERVER_IP, + upgrade_runner_environment.SMTP_SERVER_PORT, ) emailSender.send( - sender=os.environ["SMTP_EMAIL"], + sender=upgrade_runner_environment.SMTP_EMAIL, recipient=email_address, subject=subject, message=f"{type}: Failed to perform update on RSU {self.rsu_ip} due to the following error: {err}", replyEmail="", - username=os.environ["SMTP_USERNAME"], - password=os.environ["SMTP_PASSWORD"], + username=upgrade_runner_environment.SMTP_USERNAME, + password=upgrade_runner_environment.SMTP_PASSWORD, pretty=True, ) except Exception as e: diff --git a/services/addons/images/firmware_manager/upgrade_runner/yunex_upgrader.py b/services/addons/images/firmware_manager/upgrade_runner/yunex_upgrader.py index 517a5764d..2ce418adc 100644 --- a/services/addons/images/firmware_manager/upgrade_runner/yunex_upgrader.py +++ b/services/addons/images/firmware_manager/upgrade_runner/yunex_upgrader.py @@ -1,11 +1,11 @@ import upgrader import json import logging -import os import subprocess import sys import tarfile import time +from common import common_environment class YunexUpgrader(upgrader.UpgraderAbstractClass): @@ -16,7 +16,7 @@ def run_xfer_upgrade(self, file_name): xfer_command = [ "java", "-jar", - f"/home/tools/xfer_yunex.jar", + "/home/tools/xfer_yunex.jar", "-upload", file_name, f"{self.rsu_ip}:3600", @@ -124,8 +124,10 @@ def upgrade(self): # - target_firmware_version # - install_package if __name__ == "__main__": - log_level = os.environ.get("LOGGING_LEVEL", "INFO") - logging.basicConfig(format="%(levelname)s:%(message)s", level=log_level) + logging.info( + "Yunex Upgrader running with LOGGING_LEVEL: " + + str(common_environment.LOGGING_LEVEL) + ) # Trimming outer single quotes from the json.loads upgrade_info = json.loads(sys.argv[1][1:-1]) yunex_upgrader = YunexUpgrader(upgrade_info) diff --git a/services/addons/images/firmware_manager/upgrade_scheduler/upgrade_scheduler.py b/services/addons/images/firmware_manager/upgrade_scheduler/upgrade_scheduler.py index 3b965c5c9..2c5f6ce90 100644 --- a/services/addons/images/firmware_manager/upgrade_scheduler/upgrade_scheduler.py +++ b/services/addons/images/firmware_manager/upgrade_scheduler/upgrade_scheduler.py @@ -6,14 +6,10 @@ from waitress import serve import requests import logging -import os +import upgrade_scheduler_environment app = Flask(__name__) -log_level = os.environ.get("LOGGING_LEVEL", "INFO") -logging.basicConfig(format="%(levelname)s:%(message)s", level=log_level) - - # Tracker for active firmware upgrades # Key: IPv4 string of target device # Value: Dictionary with the following key-values: @@ -33,7 +29,7 @@ # Changed from a constant to a function to help with unit testing def get_upgrade_limit() -> int: try: - upgrade_limit = int(os.environ.get("ACTIVE_UPGRADE_LIMIT", "1")) + upgrade_limit = int(upgrade_scheduler_environment.ACTIVE_UPGRADE_LIMIT) return upgrade_limit except ValueError: raise ValueError( @@ -41,7 +37,6 @@ def get_upgrade_limit() -> int: ) - # Function to query the CV Manager PostgreSQL database for RSUs that have: # - A different target version than their current version # - A target firmware that complies with an existing upgrade rule relative to the RSU's current version @@ -74,26 +69,37 @@ def get_rsu_upgrade_data(rsu_ip="all"): def start_tasks_from_queue(): # Start the next process in the queue if there are less than ACTIVE_UPGRADE_LIMIT number of active upgrades occurring - while len(active_upgrades) < get_upgrade_limit() and len(upgrade_queue) > 0: + while ( + len(active_upgrades) < upgrade_scheduler_environment.ACTIVE_UPGRADE_LIMIT + and len(upgrade_queue) > 0 + ): rsu_to_upgrade = upgrade_queue.popleft() try: rsu_upgrade_info = upgrade_queue_info[rsu_to_upgrade] del upgrade_queue_info[rsu_to_upgrade] # Begin the firmware upgrade using the Upgrade Runner API - upgrade_runner_endpoint = os.environ.get("UPGRADE_RUNNER_ENDPOINT", "UNDEFINED") + upgrade_runner_endpoint = ( + upgrade_scheduler_environment.UPGRADE_RUNNER_ENDPOINT + ) if upgrade_runner_endpoint == "UNDEFINED": - raise Exception("The UPGRADE_RUNNER_ENDPOINT environment variable is undefined!") + raise Exception( + "The UPGRADE_RUNNER_ENDPOINT environment variable is undefined!" + ) - response = requests.post(f"{upgrade_runner_endpoint}/run_firmware_upgrade", json=rsu_upgrade_info) + response = requests.post( + f"{upgrade_runner_endpoint}/run_firmware_upgrade", json=rsu_upgrade_info + ) # Remove redundant ipv4_address from rsu since it is the key for active_upgrades del rsu_upgrade_info["ipv4_address"] # If the POST response includes a 201 code, add it to the active upgrades if response.status_code == 201: - logging.info(f"Firmware upgrade runner successfully requested to begin the upgrade for {rsu_to_upgrade}") + logging.info( + f"Firmware upgrade runner successfully requested to begin the upgrade for {rsu_to_upgrade}" + ) active_upgrades[rsu_to_upgrade] = rsu_upgrade_info else: logging.error( @@ -345,7 +351,7 @@ def reset_consecutive_failure_count_for_rsu(rsu_ip): def is_rsu_at_max_retries_limit(rsu_ip): - max_retries = int(os.environ.get("FW_UPGRADE_MAX_RETRY_LIMIT", "3")) + max_retries = upgrade_scheduler_environment.FW_UPGRADE_MAX_RETRY_LIMIT query_result = pgquery.query_db( f"select consecutive_failures from consecutive_firmware_upgrade_failures where rsu_id=(select rsu_id from rsus where ipv4_address='{rsu_ip}')" ) @@ -372,7 +378,9 @@ def serve_rest_api(): def init_background_task(): - logging.info("Initiating the Firmware Manager Upgrade Scheduler background checker...") + logging.info( + "Initiating the Firmware Manager Upgrade Scheduler background checker..." + ) # Run scheduler for async RSU firmware upgrade checks scheduler = BackgroundScheduler({"apscheduler.timezone": "UTC"}) scheduler.add_job(check_for_upgrades, "cron", minute="0") diff --git a/services/addons/images/firmware_manager/upgrade_scheduler/upgrade_scheduler_environment.py b/services/addons/images/firmware_manager/upgrade_scheduler/upgrade_scheduler_environment.py new file mode 100644 index 000000000..53e6c1c2b --- /dev/null +++ b/services/addons/images/firmware_manager/upgrade_scheduler/upgrade_scheduler_environment.py @@ -0,0 +1,5 @@ +from common.common_environment import get_env_var + +UPGRADE_RUNNER_ENDPOINT = get_env_var("UPGRADE_RUNNER_ENDPOINT", "UNDEFINED") +ACTIVE_UPGRADE_LIMIT = int(get_env_var("ACTIVE_UPGRADE_LIMIT", "1")) +FW_UPGRADE_MAX_RETRY_LIMIT = int(get_env_var("FW_UPGRADE_MAX_RETRY_LIMIT", "3")) diff --git a/services/addons/images/iss_health_check/iss_health_check_environment.py b/services/addons/images/iss_health_check/iss_health_check_environment.py new file mode 100644 index 000000000..18ae20ef6 --- /dev/null +++ b/services/addons/images/iss_health_check/iss_health_check_environment.py @@ -0,0 +1,21 @@ +from common.common_environment import get_env_var + + +def process_storage_type(val: str, default: str) -> str: + if not val: + return default + value = val.lower() + if value not in ["postgres", "gcp"]: + raise ValueError("Invalid STORAGE_TYPE. Must be 'postgres' or 'gcp'.") + return value + + +PROJECT_ID=get_env_var("PROJECT_ID", error=True) +ISS_API_KEY=get_env_var("ISS_API_KEY", error=True) +ISS_API_KEY_NAME=get_env_var("ISS_API_KEY_NAME", error=True) +ISS_KEY_TABLE_NAME=get_env_var("ISS_KEY_TABLE_NAME", error=True) +ISS_SCMS_TOKEN_REST_ENDPOINT=get_env_var("ISS_SCMS_TOKEN_REST_ENDPOINT", error=True) +ISS_SCMS_VEHICLE_REST_ENDPOINT=get_env_var("ISS_SCMS_VEHICLE_REST_ENDPOINT", error=True) +ISS_PROJECT_ID=get_env_var("ISS_PROJECT_ID", error=True) + +STORAGE_TYPE = process_storage_type(get_env_var("STORAGE_TYPE", warn=False), "postgres") diff --git a/services/addons/images/iss_health_check/iss_health_checker.py b/services/addons/images/iss_health_check/iss_health_checker.py index 5fa68338f..a5193701d 100644 --- a/services/addons/images/iss_health_check/iss_health_checker.py +++ b/services/addons/images/iss_health_check/iss_health_checker.py @@ -1,11 +1,11 @@ from datetime import datetime import requests import logging -import os import iss_token import common.pgquery as pgquery from dataclasses import dataclass, field from typing import Dict +import iss_health_check_environment # Set up logging @@ -68,8 +68,8 @@ def get_scms_status_data(): iss_headers["x-api-key"] = iss_token.get_token() # Create the GET request string - iss_base = os.environ["ISS_SCMS_VEHICLE_REST_ENDPOINT"] - project_id = os.environ["ISS_PROJECT_ID"] + iss_base = iss_health_check_environment.ISS_SCMS_VEHICLE_REST_ENDPOINT + project_id = iss_health_check_environment.ISS_PROJECT_ID page_size = 200 page = 0 messages_processed = 0 @@ -191,11 +191,5 @@ def validate_scms_data(value): if __name__ == "__main__": - # Configure logging based on ENV var or use default if not set - log_level = ( - "INFO" if "LOGGING_LEVEL" not in os.environ else os.environ["LOGGING_LEVEL"] - ) - logging.basicConfig(format="%(levelname)s:%(message)s", level=log_level) - scms_statuses = get_scms_status_data() insert_scms_data(scms_statuses) diff --git a/services/addons/images/iss_health_check/iss_token.py b/services/addons/images/iss_health_check/iss_token.py index 77b761d42..18bc89bcd 100644 --- a/services/addons/images/iss_health_check/iss_token.py +++ b/services/addons/images/iss_health_check/iss_token.py @@ -1,34 +1,15 @@ from google.cloud import secretmanager import common.pgquery as pgquery import requests -import os import json import uuid import logging +import iss_health_check_environment # Set up logging logger = logging.getLogger(__name__) -# Get storage type from environment variable -def get_storage_type(): - """Get the storage type for the ISS SCMS API token - """ - try : - os.environ["STORAGE_TYPE"] - except KeyError: - logger.error("STORAGE_TYPE environment variable not set, exiting") - exit(1) - - storageTypeCaseInsensitive = os.environ["STORAGE_TYPE"].casefold() - if storageTypeCaseInsensitive == "gcp": - return "gcp" - elif storageTypeCaseInsensitive == "postgres": - return "postgres" - else: - logger.error("STORAGE_TYPE environment variable not set to a valid value, exiting") - exit(1) - # GCP Secret Manager functions def create_secret(client, secret_id, parent): @@ -137,11 +118,10 @@ def add_data(table_name, common_name, token): # Main function def get_token(): - storage_type = get_storage_type() - if storage_type == "gcp": + if iss_health_check_environment.STORAGE_TYPE == "gcp": client = secretmanager.SecretManagerServiceClient() secret_id = "iss-token-secret" - parent = f"projects/{os.environ['PROJECT_ID']}" + parent = f"projects/{iss_health_check_environment.PROJECT_ID}" # Check to see if the GCP secret exists data_exists = check_if_secret_exists(client, secret_id, parent) @@ -156,11 +136,11 @@ def get_token(): # If there is no available ISS token secret, create secret logger.debug("Secret does not exist, creating secret") create_secret(client, secret_id, parent) - # Use environment variable for first run with new secret - token = os.environ["ISS_API_KEY"] - elif storage_type == "postgres": - key_table_name = os.environ["ISS_KEY_TABLE_NAME"] - + # Use iss_health_check_environment variable for first run with new secret + token = iss_health_check_environment.ISS_API_KEY + elif iss_health_check_environment.STORAGE_TYPE == "postgres": + key_table_name = iss_health_check_environment.ISS_KEY_TABLE_NAME + # check to see if data exists in the table data_exists = check_if_data_exists(key_table_name) @@ -172,17 +152,19 @@ def get_token(): token = value["token"] logger.debug(f"Received token: {friendly_name} with id {id}") else: - # if there is no data, use environment variable for first run - token = os.environ["ISS_API_KEY"] + # if there is no data, use iss_health_check_environment variable for first run + token = iss_health_check_environment.ISS_API_KEY # Pull new ISS SCMS API token - iss_base = os.environ["ISS_SCMS_TOKEN_REST_ENDPOINT"] + iss_base = iss_health_check_environment.ISS_SCMS_TOKEN_REST_ENDPOINT # Create HTTP request headers iss_headers = {"x-api-key": token} # Create the POST body - new_friendly_name = f"{os.environ['ISS_API_KEY_NAME']}_{str(uuid.uuid4())}" + new_friendly_name = ( + f"{iss_health_check_environment.ISS_API_KEY_NAME}_{str(uuid.uuid4())}" + ) iss_post_body = {"friendlyName": new_friendly_name, "expireDays": 1} # Create new ISS SCMS API Token to ensure its freshness @@ -203,10 +185,10 @@ def get_token(): version_data = {"name": new_friendly_name, "token": new_token} - if get_storage_type() == "gcp": + if iss_health_check_environment.STORAGE_TYPE == "gcp": # Add new version to the secret add_secret_version(client, secret_id, parent, version_data) - elif get_storage_type() == "postgres": + elif iss_health_check_environment.STORAGE_TYPE == "postgres": # add new entry to the table add_data(key_table_name, new_friendly_name, new_token) diff --git a/services/addons/images/obu_ota_server/commsignia_manifest.py b/services/addons/images/obu_ota_server/commsignia_manifest.py index e6d368d27..96f71b9d3 100644 --- a/services/addons/images/obu_ota_server/commsignia_manifest.py +++ b/services/addons/images/obu_ota_server/commsignia_manifest.py @@ -2,7 +2,6 @@ import datetime import re from typing import List -import os import copy document = { diff --git a/services/addons/images/obu_ota_server/obu_ota_server.py b/services/addons/images/obu_ota_server/obu_ota_server.py index 06386b8fd..8b29405b4 100644 --- a/services/addons/images/obu_ota_server/obu_ota_server.py +++ b/services/addons/images/obu_ota_server/obu_ota_server.py @@ -2,26 +2,24 @@ from fastapi import FastAPI, Request, Response, HTTPException, Depends from fastapi.security import HTTPBasic, HTTPBasicCredentials from common import gcs_utils, pgquery -import commsignia_manifest -import os +from addons.images.obu_ota_server import commsignia_manifest import glob +import os import aiofiles -from starlette.responses import Response import logging from datetime import datetime import asyncio +import obu_ota_server_environment app = FastAPI() -log_level = "INFO" if "LOGGING_LEVEL" not in os.environ else os.environ["LOGGING_LEVEL"] -logging.basicConfig(format="%(levelname)s:%(message)s", level=log_level) security = HTTPBasic() commsignia_file_ext = ".tar.sig" def authenticate_user(credentials: HTTPBasicCredentials = Depends(security)) -> str: - correct_username = os.getenv("OTA_USERNAME") - correct_password = os.getenv("OTA_PASSWORD") + correct_username = obu_ota_server_environment.OTA_USERNAME + correct_password = obu_ota_server_environment.OTA_PASSWORD if ( credentials.username != correct_username or credentials.password != correct_password @@ -43,20 +41,20 @@ async def read_root(request: Request): def get_firmware_list() -> list: - blob_storage_provider = os.getenv("BLOB_STORAGE_PROVIDER", "DOCKER") + blob_storage_provider = obu_ota_server_environment.BLOB_STORAGE_PROVIDER files = [] file_extension = commsignia_file_ext if blob_storage_provider.upper() == "DOCKER": files = glob.glob(f"/firmwares/*{file_extension}") elif blob_storage_provider.upper() == "GCP": - blob_storage_path = os.getenv("BLOB_STORAGE_PATH", "DOCKER") + blob_storage_path = obu_ota_server_environment.BLOB_STORAGE_PATH files = gcs_utils.list_gcs_blobs(blob_storage_path, file_extension) return files def get_host_name() -> str: - host_name = os.getenv("SERVER_HOST", "localhost") - tls_enabled = os.getenv("NGINX_ENCRYPTION", "plain") + host_name = obu_ota_server_environment.SERVER_HOST + tls_enabled = obu_ota_server_environment.NGINX_ENCRYPTION if tls_enabled.lower() == "ssl": host_name = "https://" + host_name else: @@ -78,7 +76,7 @@ async def get_manifest(request: Request) -> dict[str, Any]: def get_firmware(firmware_id: str, local_file_path: str) -> bool: try: - blob_storage_provider = os.getenv("BLOB_STORAGE_PROVIDER", "DOCKER") + blob_storage_provider = obu_ota_server_environment.BLOB_STORAGE_PROVIDER # checks if firmware exists locally if not os.path.exists(local_file_path): # If configured to only use local storage, return False as firmware does not exist @@ -131,12 +129,11 @@ async def read_file( def removed_old_logs(serialnum: str): try: - max_count = int(os.getenv("MAX_COUNT", 10)) success_count = pgquery.query_db( f"SELECT COUNT(*) FROM public.obu_ota_requests WHERE obu_sn = '{serialnum}' AND error_status = B'0'" ) - if success_count[0][0] > max_count: - excess_count = success_count[0][0] - max_count + if success_count[0][0] > obu_ota_server_environment.MAX_COUNT: + excess_count = success_count[0][0] - obu_ota_server_environment.MAX_COUNT oldest_entries = pgquery.query_db( f"SELECT request_id FROM public.obu_ota_requests WHERE obu_sn = '{serialnum}' AND error_status = B'0' ORDER BY request_datetime ASC LIMIT {excess_count}" ) @@ -200,7 +197,7 @@ async def get_fw(request: Request, firmware_id: str): content=data, media_type="application/octet-stream", headers=headers ) except Exception as e: - asyncio.create_task(log_request(1, request, firmware_id, 1, e.detail)) + asyncio.create_task(log_request(1, request, firmware_id, 1, str(e))) logging.error( f"get_fw: Error responding with firmware with error: {e} for firmware_id: {firmware_id}" ) diff --git a/services/addons/images/obu_ota_server/obu_ota_server_environment.py b/services/addons/images/obu_ota_server/obu_ota_server_environment.py new file mode 100644 index 000000000..c79a3ec61 --- /dev/null +++ b/services/addons/images/obu_ota_server/obu_ota_server_environment.py @@ -0,0 +1,9 @@ +from common.common_environment import get_env_var + +OTA_USERNAME=get_env_var("OTA_USERNAME", "admin") +OTA_PASSWORD=get_env_var("OTA_PASSWORD", error=True) +BLOB_STORAGE_PROVIDER=get_env_var("BLOB_STORAGE_PROVIDER", "DOCKER") +BLOB_STORAGE_PATH=get_env_var("BLOB_STORAGE_PATH", "DOCKER") +SERVER_HOST=get_env_var("SERVER_HOST", "localhost") +NGINX_ENCRYPTION=get_env_var("NGINX_ENCRYPTION", "plain", warn=False) +MAX_COUNT=int(get_env_var("MAX_COUNT", "10", warn=False)) diff --git a/services/addons/images/rsu_status_check/purger.py b/services/addons/images/rsu_status_check/purger.py index 7b1ec0a35..fbfc8a8d3 100644 --- a/services/addons/images/rsu_status_check/purger.py +++ b/services/addons/images/rsu_status_check/purger.py @@ -1,7 +1,8 @@ -from datetime import datetime, timedelta, timezone -import os +from datetime import datetime, timedelta import logging import common.pgquery as pgquery +import rsu_status_check_environment +from common import common_environment def get_all_rsus(): @@ -83,17 +84,12 @@ def purge_ping_data(stale_period): if __name__ == "__main__": - # Configure logging based on ENV var or use default if not set - log_level = os.environ.get("LOGGING_LEVEL", "INFO") - logging.basicConfig(format="%(levelname)s:%(message)s", level=log_level) - run_service = ( - os.environ.get("RSU_PING", "False").lower() == "true" - or os.environ.get("ZABBIX", "False").lower() == "true" + rsu_status_check_environment.RSU_PING or rsu_status_check_environment.ZABBIX ) if not run_service: logging.info("The purger service is disabled and will not run") exit() - stale_period = int(os.environ["STALE_PERIOD"]) + stale_period = rsu_status_check_environment.STALE_PERIOD_HOURS purge_ping_data(stale_period) diff --git a/services/addons/images/rsu_status_check/rsu_msgfwd_fetch.py b/services/addons/images/rsu_status_check/rsu_msgfwd_fetch.py index 7b7487974..66017eab0 100644 --- a/services/addons/images/rsu_status_check/rsu_msgfwd_fetch.py +++ b/services/addons/images/rsu_status_check/rsu_msgfwd_fetch.py @@ -1,20 +1,14 @@ -import os import logging from common.snmp.update_pg.update_rsu_message_forward import ( UpdatePostgresRsuMessageForward, ) +import rsu_status_check_environment # Pulls the latest message forwarding configuration information from all RSUs in the PostgreSQL database # through SNMP and updates the PostgreSQL database with the latest information def main(): - # Configure logging based on ENV var or use default if not set - log_level = os.environ.get("LOGGING_LEVEL", "INFO") - log_level = "INFO" if log_level == "" else log_level - logging.basicConfig(format="%(levelname)s:%(message)s", level=log_level) - - run_service = os.environ.get("RSU_MSGFWD_FETCH", "False").lower() == "true" - if not run_service: + if not rsu_status_check_environment.RSU_MSGFWD_FETCH: logging.info("The rsu-msgfwd-fetch service is disabled and will not run") return diff --git a/services/addons/images/rsu_status_check/rsu_ping_fetch.py b/services/addons/images/rsu_status_check/rsu_ping_fetch.py index 5393e25fd..d6b718855 100644 --- a/services/addons/images/rsu_status_check/rsu_ping_fetch.py +++ b/services/addons/images/rsu_status_check/rsu_ping_fetch.py @@ -1,7 +1,7 @@ import requests -import os import logging import common.pgquery as pgquery +import rsu_status_check_environment def get_rsu_data(): @@ -37,7 +37,7 @@ def insert_rsu_ping(request_json): class RsuStatusFetch: def __init__(self): - self.ZABBIX_ENDPOINT = os.environ["ZABBIX_ENDPOINT"] + self.ZABBIX_ENDPOINT = rsu_status_check_environment.ZABBIX_ENDPOINT self.ZABBIX_AUTH = "" def setZabbixAuth(self): @@ -47,8 +47,8 @@ def setZabbixAuth(self): "method": "user.login", "id": 1, "params": { - "username": os.environ["ZABBIX_USER"], - "password": os.environ["ZABBIX_PASSWORD"], + "username": rsu_status_check_environment.ZABBIX_USER, + "password": rsu_status_check_environment.ZABBIX_PASSWORD, }, } @@ -144,14 +144,8 @@ def run(self): if __name__ == "__main__": - # Configure logging based on ENV var or use default if not set - log_level = os.environ.get("LOGGING_LEVEL", "INFO") - log_level = "INFO" if log_level == "" else log_level - logging.basicConfig(format="%(levelname)s:%(message)s", level=log_level) - run_service = ( - os.environ.get("RSU_PING", "False").lower() == "true" - and os.environ.get("ZABBIX", "False").lower() == "true" + rsu_status_check_environment.RSU_PING and rsu_status_check_environment.ZABBIX ) if not run_service: logging.info("The rsu-ping-fetch service is disabled and will not run") diff --git a/services/addons/images/rsu_status_check/rsu_pinger.py b/services/addons/images/rsu_status_check/rsu_pinger.py index db403a30b..0084312b0 100644 --- a/services/addons/images/rsu_status_check/rsu_pinger.py +++ b/services/addons/images/rsu_status_check/rsu_pinger.py @@ -1,9 +1,9 @@ -import os import logging import time import common.pgquery as pgquery from datetime import datetime from subprocess import Popen, DEVNULL +import rsu_status_check_environment def insert_ping_data(ping_data, ping_time): @@ -83,14 +83,8 @@ def run_rsu_pinger(): if __name__ == "__main__": - # Configure logging based on ENV var or use default if not set - log_level = os.environ.get("LOGGING_LEVEL", "INFO") - log_level = "INFO" if log_level == "" else log_level - logging.basicConfig(format="%(levelname)s:%(message)s", level=log_level) - run_service = ( - os.environ.get("RSU_PING", "False").lower() == "true" - and os.environ.get("ZABBIX", "False").lower() == "false" + rsu_status_check_environment.RSU_PING and rsu_status_check_environment.ZABBIX ) if not run_service: logging.info("The rsu-pinger service is disabled and will not run") diff --git a/services/addons/images/rsu_status_check/rsu_status_check_environment.py b/services/addons/images/rsu_status_check/rsu_status_check_environment.py new file mode 100644 index 000000000..455eb4b09 --- /dev/null +++ b/services/addons/images/rsu_status_check/rsu_status_check_environment.py @@ -0,0 +1,10 @@ +from common.common_environment import get_env_var + +RSU_PING=get_env_var("RSU_PING", "False", warn=False).lower() == "true" +ZABBIX=get_env_var("ZABBIX", "False", warn=False).lower() == "true" +STALE_PERIOD_HOURS=int(get_env_var("STALE_PERIOD", "24", warn=False)) +RSU_MSGFWD_FETCH=get_env_var("RSU_MSGFWD_FETCH", "False", warn=False).lower() == "true" + +ZABBIX_ENDPOINT=get_env_var("ZABBIX_ENDPOINT", error=True) +ZABBIX_USER=get_env_var("ZABBIX_USER", error=True) +ZABBIX_PASSWORD=get_env_var("ZABBIX_PASSWORD", error=True) diff --git a/services/addons/tests/count_metric/conftest.py b/services/addons/tests/count_metric/conftest.py new file mode 100644 index 000000000..d352f79d5 --- /dev/null +++ b/services/addons/tests/count_metric/conftest.py @@ -0,0 +1,6 @@ +import os + +os.environ['SMTP_SERVER_IP'] = 'smtp-server-ip' +os.environ['SMTP_EMAIL'] = 'smtp-email' +os.environ['SMTP_USERNAME'] = 'smtp-username' +os.environ['SMTP_PASSWORD'] = 'smtp-password' diff --git a/services/addons/tests/count_metric/test_daily_emailer.py b/services/addons/tests/count_metric/test_daily_emailer.py index 84b57c7fd..b02030d36 100644 --- a/services/addons/tests/count_metric/test_daily_emailer.py +++ b/services/addons/tests/count_metric/test_daily_emailer.py @@ -198,16 +198,11 @@ def test_prepare_org_rsu_dict(mock_query_db): daily_emailer.message_types = ["BSM", "TIM", "Map", "SPaT", "SRM", "SSM"] -@patch.dict( - os.environ, - { - "DEPLOYMENT_TITLE": "Test", - "SMTP_SERVER_IP": "10.0.0.1", - "SMTP_USERNAME": "username", - "SMTP_PASSWORD": "password", - "SMTP_EMAIL": "test@gmail.com", - }, -) +@patch("count_metric_environment.DEPLOYMENT_TITLE", "Test") +@patch("count_metric_environment.SMTP_SERVER_IP", "10.0.0.1") +@patch("count_metric_environment.SMTP_USERNAME", "username") +@patch("count_metric_environment.SMTP_PASSWORD", "password") +@patch("count_metric_environment.SMTP_EMAIL", "test@gmail.com") @patch("addons.images.count_metric.daily_emailer.EmailSender") @patch("addons.images.count_metric.daily_emailer.get_email_list") def test_email_daily_counts(mock_email_list, mock_emailsender): @@ -228,13 +223,8 @@ def test_email_daily_counts(mock_email_list, mock_emailsender): ) -@patch.dict( - os.environ, - { - "MONGO_DB_URI": "mongo-uri", - "MONGO_DB_NAME": "test_db", - }, -) +@patch("count_metric_environment.MONGO_DB_URI", "mongo-uri") +@patch("count_metric_environment.MONGO_DB_NAME", "test_db") @patch("addons.images.count_metric.daily_emailer.MongoClient", MagicMock()) @patch("addons.images.count_metric.daily_emailer.gen_email.generate_email_body") @patch("addons.images.count_metric.daily_emailer.email_daily_counts") diff --git a/services/addons/tests/count_metric/test_gen_email.py b/services/addons/tests/count_metric/test_gen_email.py index 98196d607..9fb2f4c9f 100644 --- a/services/addons/tests/count_metric/test_gen_email.py +++ b/services/addons/tests/count_metric/test_gen_email.py @@ -113,10 +113,7 @@ def test_generate_count_table_empty(): assert result == "" -@patch.dict( - os.environ, - {"DEPLOYMENT_TITLE": "Test"}, -) +@patch("count_metric_environment.DEPLOYMENT_TITLE", "Test") @patch("addons.images.count_metric.gen_email.generate_count_table") def test_generate_email_body(mock_generate_count_table): mock_generate_count_table.return_value = "" diff --git a/services/addons/tests/count_metric/test_mongo_counter.py b/services/addons/tests/count_metric/test_mongo_counter.py index 3cab00fc9..e34a49628 100644 --- a/services/addons/tests/count_metric/test_mongo_counter.py +++ b/services/addons/tests/count_metric/test_mongo_counter.py @@ -1,5 +1,4 @@ -import os -from datetime import datetime, timedelta +from datetime import datetime from mock import MagicMock, patch from addons.images.count_metric import mongo_counter @@ -22,7 +21,6 @@ def test_write_counts_empty(): mock_collection.insert_many.assert_not_called() -@patch.dict(os.environ, {"MONGO_DB_URI": "uri", "MONGO_DB_NAME": "name"}) def test_count_query_bsm(): mock_collection = MagicMock() mock_collection.aggregate.return_value = [ @@ -53,7 +51,6 @@ def test_count_query_bsm(): assert result == expected_result -@patch.dict(os.environ, {"MONGO_DB_URI": "uri", "MONGO_DB_NAME": "name"}) @patch("addons.images.count_metric.mongo_counter.write_counts") @patch("addons.images.count_metric.mongo_counter.count_query") def test_run_mongo_counter(mock_count_query, mock_write_counts): diff --git a/services/addons/tests/firmware_manager/upgrade_runner/test_upgrader.py b/services/addons/tests/firmware_manager/upgrade_runner/test_upgrader.py index 3589e4f21..2391c42f8 100644 --- a/services/addons/tests/firmware_manager/upgrade_runner/test_upgrader.py +++ b/services/addons/tests/firmware_manager/upgrade_runner/test_upgrader.py @@ -74,7 +74,10 @@ def test_cleanup_not_exist(mock_Path, mock_shutil): mock_shutil.rmtree.assert_not_called() -@patch.dict(os.environ, {"BLOB_STORAGE_PROVIDER": "GCP"}) +@patch( + "upgrade_runner_environment.BLOB_STORAGE_PROVIDER", + "GCP", +) @patch("common.gcs_utils.download_gcp_blob") @patch("addons.images.firmware_manager.upgrade_runner.upgrader.Path") def test_download_blob_gcp(mock_Path, mock_download_gcp_blob): @@ -91,7 +94,10 @@ def test_download_blob_gcp(mock_Path, mock_download_gcp_blob): ) -@patch.dict(os.environ, {"BLOB_STORAGE_PROVIDER": "DOCKER"}) +@patch( + "upgrade_runner_environment.BLOB_STORAGE_PROVIDER", + "DOCKER", +) @patch( "addons.images.firmware_manager.upgrade_runner.upgrader.download_blob.download_docker_blob" ) @@ -109,7 +115,10 @@ def test_download_blob_docker(mock_Path, mock_download_docker_blob): ) -@patch.dict(os.environ, {"BLOB_STORAGE_PROVIDER": "Test"}) +@patch( + "upgrade_runner_environment.BLOB_STORAGE_PROVIDER", + "Test", +) @patch("addons.images.firmware_manager.upgrade_runner.upgrader.logging") @patch("common.gcs_utils.download_gcp_blob") @patch("addons.images.firmware_manager.upgrade_runner.upgrader.Path") @@ -125,7 +134,10 @@ def test_download_blob_not_supported(mock_Path, mock_download_gcp_blob, mock_log mock_logging.error.assert_called_with("Unsupported blob storage provider") -@patch.dict("os.environ", {"UPGRADE_SCHEDULER_ENDPOINT": "http://test-endpoint"}) +@patch( + "upgrade_runner_environment.UPGRADE_SCHEDULER_ENDPOINT", + "http://test-endpoint", +) @patch("addons.images.firmware_manager.upgrade_runner.upgrader.logging") @patch("addons.images.firmware_manager.upgrade_runner.upgrader.requests") def test_notify_firmware_manager_success(mock_requests, mock_logging): @@ -142,7 +154,10 @@ def test_notify_firmware_manager_success(mock_requests, mock_logging): mock_requests.post.assert_called_with(expected_url, json=expected_body) -@patch.dict("os.environ", {"UPGRADE_SCHEDULER_ENDPOINT": "http://test-endpoint"}) +@patch( + "upgrade_runner_environment.UPGRADE_SCHEDULER_ENDPOINT", + "http://test-endpoint", +) @patch("addons.images.firmware_manager.upgrade_runner.upgrader.logging") @patch("addons.images.firmware_manager.upgrade_runner.upgrader.requests") def test_notify_firmware_manager_fail(mock_requests, mock_logging): diff --git a/services/addons/tests/firmware_manager/upgrade_scheduler/conftest.py b/services/addons/tests/firmware_manager/upgrade_scheduler/conftest.py new file mode 100644 index 000000000..d352f79d5 --- /dev/null +++ b/services/addons/tests/firmware_manager/upgrade_scheduler/conftest.py @@ -0,0 +1,6 @@ +import os + +os.environ['SMTP_SERVER_IP'] = 'smtp-server-ip' +os.environ['SMTP_EMAIL'] = 'smtp-email' +os.environ['SMTP_USERNAME'] = 'smtp-username' +os.environ['SMTP_PASSWORD'] = 'smtp-password' diff --git a/services/addons/tests/firmware_manager/upgrade_scheduler/test_upgrade_scheduler.py b/services/addons/tests/firmware_manager/upgrade_scheduler/test_upgrade_scheduler.py index 4a10f58f2..878561673 100644 --- a/services/addons/tests/firmware_manager/upgrade_scheduler/test_upgrade_scheduler.py +++ b/services/addons/tests/firmware_manager/upgrade_scheduler/test_upgrade_scheduler.py @@ -42,7 +42,10 @@ def test_get_rsu_upgrade_data_one(mock_querydb): # start_tasks_from_queue tests -@patch.dict("os.environ", {"UPGRADE_RUNNER_ENDPOINT": "http://test-endpoint"}) +@patch( + "upgrade_scheduler_environment.UPGRADE_RUNNER_ENDPOINT", + "http://test-endpoint", +) @patch( "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", {}, @@ -133,7 +136,14 @@ def test_start_tasks_from_queue_no_env_var(mock_post, mock_logging): ) -@patch.dict("os.environ", {"UPGRADE_RUNNER_ENDPOINT": "http://test-endpoint"}) +@patch( + "upgrade_scheduler_environment.UPGRADE_RUNNER_ENDPOINT", + "http://test-endpoint", +) +@patch( + "upgrade_scheduler_environment.UPGRADE_RUNNER_ENDPOINT", + "http://test-endpoint", +) @patch( "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", {}, @@ -188,7 +198,10 @@ def test_start_tasks_from_queue_post_success(mock_post, mock_logging): mock_logging.error.assert_not_called() -@patch.dict("os.environ", {"UPGRADE_RUNNER_ENDPOINT": "http://test-endpoint"}) +@patch( + "upgrade_scheduler_environment.UPGRADE_RUNNER_ENDPOINT", + "http://test-endpoint", +) @patch( "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", {}, @@ -845,7 +858,10 @@ def test_list_active_upgrades(mock_logging): # check_for_upgrades tests -@patch.dict("os.environ", {"UPGRADE_RUNNER_ENDPOINT": "http://test-endpoint"}) +@patch( + "upgrade_scheduler_environment.UPGRADE_RUNNER_ENDPOINT", + "http://test-endpoint", +) @patch( "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.was_latest_ping_successful_for_rsu" ) @@ -1007,7 +1023,10 @@ def test_reset_consecutive_failure_count_for_rsu(mock_write_db): mock_write_db.assert_called_with(expected_query) -@patch.dict("os.environ", {"FW_UPGRADE_MAX_RETRY_LIMIT": "3"}) +@patch( + "upgrade_scheduler_environment.FW_UPGRADE_MAX_RETRY_LIMIT", + 3, +) @patch( "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.pgquery.query_db" ) @@ -1021,11 +1040,14 @@ def test_is_rsu_at_max_retries_limit_TRUE(mock_query_db): result = upgrade_scheduler.is_rsu_at_max_retries_limit(rsu_ip) # verify - assert result == True + assert result is True mock_query_db.assert_called_with(expected_query) -@patch.dict("os.environ", {"FW_UPGRADE_MAX_RETRY_LIMIT": "3"}) +@patch( + "upgrade_scheduler_environment.FW_UPGRADE_MAX_RETRY_LIMIT", + 3, +) @patch( "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.pgquery.query_db" ) @@ -1039,11 +1061,14 @@ def test_is_rsu_at_max_retries_limit_FALSE(mock_query_db): result = upgrade_scheduler.is_rsu_at_max_retries_limit(rsu_ip) # verify - assert result == False + assert result is False mock_query_db.assert_called_with(expected_query) -@patch.dict("os.environ", {"FW_UPGRADE_MAX_RETRY_LIMIT": "3"}) +@patch( + "upgrade_scheduler_environment.FW_UPGRADE_MAX_RETRY_LIMIT", + 3, +) @patch( "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.pgquery.query_db" ) @@ -1057,7 +1082,7 @@ def test_is_rsu_at_max_retries_limit_NO_RESULTS(mock_query_db): result = upgrade_scheduler.is_rsu_at_max_retries_limit(rsu_ip) # verify - assert result == False + assert result is False mock_query_db.assert_called_with(expected_query) @@ -1099,21 +1124,10 @@ def test_init_background_task(mock_bgscheduler): mock_bgscheduler_obj.start.assert_called_with() -def test_get_upgrade_limit_no_env(): - limit = upgrade_scheduler.get_upgrade_limit() - assert limit == 1 - - -@patch.dict("os.environ", {"ACTIVE_UPGRADE_LIMIT": "5"}) +@patch( + "upgrade_scheduler_environment.ACTIVE_UPGRADE_LIMIT", + 5, +) def test_get_upgrade_limit_with_env(): limit = upgrade_scheduler.get_upgrade_limit() assert limit == 5 - - -@patch.dict("os.environ", {"ACTIVE_UPGRADE_LIMIT": "bad_value"}) -def test_get_upgrade_limit_with_bad_env(): - with pytest.raises( - ValueError, - match="The environment variable 'ACTIVE_UPGRADE_LIMIT' must be an integer.", - ): - upgrade_scheduler.get_upgrade_limit() diff --git a/services/addons/tests/iss_health_check/conftest.py b/services/addons/tests/iss_health_check/conftest.py new file mode 100644 index 000000000..f03fbd385 --- /dev/null +++ b/services/addons/tests/iss_health_check/conftest.py @@ -0,0 +1,9 @@ +import os + +os.environ['PROJECT_ID'] = 'project-id' +os.environ['ISS_API_KEY'] = 'iss-api-key' +os.environ['ISS_API_KEY_NAME'] = 'iss-api-key-name' +os.environ['ISS_KEY_TABLE_NAME'] = 'iss-key-table-name' +os.environ['ISS_SCMS_TOKEN_REST_ENDPOINT'] = 'iss-scms-token-rest-endpoint' +os.environ['ISS_SCMS_VEHICLE_REST_ENDPOINT'] = 'iss-scms-vehicle-rest-endpoint' +os.environ['ISS_PROJECT_ID'] = 'iss-project-id' diff --git a/services/addons/tests/iss_health_check/test_environment_iss_health_check.py b/services/addons/tests/iss_health_check/test_environment_iss_health_check.py new file mode 100644 index 000000000..6dd5ddd94 --- /dev/null +++ b/services/addons/tests/iss_health_check/test_environment_iss_health_check.py @@ -0,0 +1,35 @@ +import pytest +from addons.images.iss_health_check import iss_health_check_environment + + +def test_get_storage_type_gcp(): + actual_value = iss_health_check_environment.process_storage_type("gcp", "postgres") + assert actual_value == "gcp" + + +def test_get_storage_type_postgres(): + actual_value = iss_health_check_environment.process_storage_type( + "postgres", "postgres" + ) + assert actual_value == "postgres" + + +def test_get_storage_type_gcp_case_insensitive(): + actual_value = iss_health_check_environment.process_storage_type("GCP", "postgres") + assert actual_value == "gcp" + + +def test_get_storage_type_postgres_case_insensitive(): + actual_value = iss_health_check_environment.process_storage_type( + "POSTGRES", "postgres" + ) + assert actual_value == "postgres" + + +def test_get_storage_type_invalid(): + with pytest.raises(ValueError): + iss_health_check_environment.process_storage_type("test", "postgres") + + +def test_get_storage_type_unset(): + iss_health_check_environment.process_storage_type(None, "postgres") == "postgres" diff --git a/services/addons/tests/iss_health_check/test_iss_health_checker.py b/services/addons/tests/iss_health_check/test_iss_health_checker.py index 0e2069d32..658f5bcb9 100644 --- a/services/addons/tests/iss_health_check/test_iss_health_checker.py +++ b/services/addons/tests/iss_health_check/test_iss_health_checker.py @@ -1,5 +1,4 @@ from unittest.mock import patch -import os from addons.images.iss_health_check import iss_health_checker from addons.images.iss_health_check.iss_health_checker import RsuDataWrapper @@ -39,14 +38,12 @@ def test_get_rsu_data_with_data(mock_query_db): ) -@patch.dict( - os.environ, - { - "ISS_API_KEY": "test", - "ISS_SCMS_VEHICLE_REST_ENDPOINT": "https://api.dm.iss-scms.com/api/test", - "ISS_PROJECT_ID": "test", - }, +@patch("iss_health_check_environment.ISS_API_KEY", "test") +@patch( + "iss_health_check_environment.ISS_SCMS_VEHICLE_REST_ENDPOINT", + "https://api.dm.iss-scms.com/api/test", ) +@patch("iss_health_check_environment.ISS_PROJECT_ID", "test") @patch("addons.images.iss_health_check.iss_health_checker.requests.Response") @patch("addons.images.iss_health_check.iss_health_checker.requests") @patch("addons.images.iss_health_check.iss_health_checker.iss_token") @@ -205,4 +202,4 @@ def test_insert_scms_data_no_expiration(mock_write_db, mock_datetime): 'INSERT INTO public.scms_health("timestamp", health, expiration, rsu_id) VALUES ' "('2022-11-03T00:00:00.000Z', '0', 'test', 2)" ) - mock_write_db.assert_called_with(expectedQuery) \ No newline at end of file + mock_write_db.assert_called_with(expectedQuery) diff --git a/services/addons/tests/iss_health_check/test_iss_token.py b/services/addons/tests/iss_health_check/test_iss_token.py index 4e588dd6c..32334854a 100644 --- a/services/addons/tests/iss_health_check/test_iss_token.py +++ b/services/addons/tests/iss_health_check/test_iss_token.py @@ -1,77 +1,9 @@ from unittest.mock import patch, MagicMock -import os import json -import pytest - from addons.images.iss_health_check import iss_token -# --------------------- Storage Type tests --------------------- -@patch.dict( - os.environ, - { - "STORAGE_TYPE": "gcp", - }, -) -def test_get_storage_type_gcp(): - actual_value = iss_token.get_storage_type() - assert actual_value == "gcp" - - -@patch.dict( - os.environ, - { - "STORAGE_TYPE": "postgres", - }, -) -def test_get_storage_type_postgres(): - actual_value = iss_token.get_storage_type() - assert actual_value == "postgres" - - -@patch.dict( - os.environ, - { - "STORAGE_TYPE": "GCP", - }, -) -def test_get_storage_type_gcp_case_insensitive(): - actual_value = iss_token.get_storage_type() - assert actual_value == "gcp" - - -@patch.dict( - os.environ, - { - "STORAGE_TYPE": "POSTGRES", - }, -) -def test_get_storage_type_postgres_case_insensitive(): - actual_value = iss_token.get_storage_type() - assert actual_value == "postgres" - - -@patch.dict( - os.environ, - { - "STORAGE_TYPE": "test", - }, -) -def test_get_storage_type_invalid(): - with pytest.raises(SystemExit): - iss_token.get_storage_type() - - -@patch.dict(os.environ, {}, clear=True) -def test_get_storage_type_unset(): - with pytest.raises(SystemExit): - iss_token.get_storage_type() - - -# --------------------- end of Storage Type tests --------------------- - - # --------------------- GCP tests --------------------- @patch( "addons.images.iss_health_check.iss_token.secretmanager.SecretManagerServiceClient" @@ -103,7 +35,7 @@ def test_check_if_secret_exists_true(mock_secretmanager, mock_sm_client): ) mock_secretmanager.ListSecretsRequest.assert_called_with(parent="test-parent") mock_sm_client.list_secrets.assert_called_with(request="list-request") - assert actual_value == True + assert actual_value is True @patch( @@ -123,7 +55,7 @@ def test_check_if_secret_exists_false(mock_secretmanager, mock_sm_client): ) mock_secretmanager.ListSecretsRequest.assert_called_with(parent="test-parent") mock_sm_client.list_secrets.assert_called_with(request="list-request") - assert actual_value == False + assert actual_value is False @patch( @@ -159,16 +91,14 @@ def test_add_secret_version(mock_sm_client): mock_sm_client.add_secret_version.assert_called_with(request=expected_request) -@patch.dict( - os.environ, - { - "PROJECT_ID": "test-proj", - "ISS_API_KEY": "test-api-key", - "ISS_SCMS_TOKEN_REST_ENDPOINT": "https://api.dm.iss-scms.com/api/test-token", - "ISS_API_KEY_NAME": "test-api-key-name", - "STORAGE_TYPE": "gcp", - }, +@patch("iss_health_check_environment.PROJECT_ID", "test-proj") +@patch("iss_health_check_environment.ISS_API_KEY", "test-api-key") +@patch( + "iss_health_check_environment.ISS_SCMS_TOKEN_REST_ENDPOINT", + "https://api.dm.iss-scms.com/api/test-token", ) +@patch("iss_health_check_environment.ISS_API_KEY_NAME", "test-api-key-name") +@patch("iss_health_check_environment.STORAGE_TYPE", "gcp") @patch("addons.images.iss_health_check.iss_token.requests.Response") @patch("addons.images.iss_health_check.iss_token.requests") @patch("addons.images.iss_health_check.iss_token.uuid") @@ -224,16 +154,14 @@ def test_get_token_create_secret( assert actual_value == expected_value -@patch.dict( - os.environ, - { - "PROJECT_ID": "test-proj", - "ISS_API_KEY": "test-api-key", - "ISS_SCMS_TOKEN_REST_ENDPOINT": "https://api.dm.iss-scms.com/api/test-token", - "ISS_API_KEY_NAME": "test-api-key-name", - "STORAGE_TYPE": "gcp", - }, +@patch("iss_health_check_environment.PROJECT_ID", "test-proj") +@patch("iss_health_check_environment.ISS_API_KEY", "test-api-key") +@patch( + "iss_health_check_environment.ISS_SCMS_TOKEN_REST_ENDPOINT", + "https://api.dm.iss-scms.com/api/test-token", ) +@patch("iss_health_check_environment.ISS_API_KEY_NAME", "test-api-key-name") +@patch("iss_health_check_environment.STORAGE_TYPE", "gcp") @patch("addons.images.iss_health_check.iss_token.requests.Response") @patch("addons.images.iss_health_check.iss_token.requests") @patch("addons.images.iss_health_check.iss_token.uuid") @@ -312,7 +240,7 @@ def test_check_if_data_exists_true(mock_pgquery): actual_value = iss_token.check_if_data_exists("test-table-name") expected_query = "SELECT * FROM test-table-name" mock_pgquery.query_db.assert_called_with(expected_query) - assert actual_value == True + assert actual_value is True @patch( @@ -323,7 +251,7 @@ def test_check_if_data_exists_false(mock_pgquery): actual_value = iss_token.check_if_data_exists("test-table-name") expected_query = "SELECT * FROM test-table-name" mock_pgquery.query_db.assert_called_with(expected_query) - assert actual_value == False + assert actual_value is False @patch( @@ -349,17 +277,15 @@ def test_get_latest_data(mock_pgquery): assert actual_value == {"id": 1, "name": "test-common-name", "token": "test-token"} -@patch.dict( - os.environ, - { - "PROJECT_ID": "test-proj", - "ISS_API_KEY": "test-api-key", - "ISS_SCMS_TOKEN_REST_ENDPOINT": "https://api.dm.iss-scms.com/api/test-token", - "ISS_API_KEY_NAME": "test-api-key-name", - "STORAGE_TYPE": "postgres", - "ISS_KEY_TABLE_NAME": "test-table-name", - }, +@patch("iss_health_check_environment.PROJECT_ID", "test-proj") +@patch("iss_health_check_environment.ISS_API_KEY", "test-api-key") +@patch( + "iss_health_check_environment.ISS_SCMS_TOKEN_REST_ENDPOINT", + "https://api.dm.iss-scms.com/api/test-token", ) +@patch("iss_health_check_environment.ISS_API_KEY_NAME", "test-api-key-name") +@patch("iss_health_check_environment.STORAGE_TYPE", "postgres") +@patch("iss_health_check_environment.ISS_KEY_TABLE_NAME", "test-table-name") @patch("addons.images.iss_health_check.iss_token.requests.Response") @patch("addons.images.iss_health_check.iss_token.requests") @patch("addons.images.iss_health_check.iss_token.uuid") @@ -400,17 +326,15 @@ def test_get_token_data_does_not_exist( assert result == "new-iss-token" -@patch.dict( - os.environ, - { - "PROJECT_ID": "test-proj", - "ISS_API_KEY": "test-api-key", - "ISS_SCMS_TOKEN_REST_ENDPOINT": "https://api.dm.iss-scms.com/api/test-token", - "ISS_API_KEY_NAME": "test-api-key-name", - "STORAGE_TYPE": "postgres", - "ISS_KEY_TABLE_NAME": "test-table-name", - }, +@patch("iss_health_check_environment.PROJECT_ID", "test-proj") +@patch("iss_health_check_environment.ISS_API_KEY", "test-api-key") +@patch( + "iss_health_check_environment.ISS_SCMS_TOKEN_REST_ENDPOINT", + "https://api.dm.iss-scms.com/api/test-token", ) +@patch("iss_health_check_environment.ISS_API_KEY_NAME", "test-api-key-name") +@patch("iss_health_check_environment.STORAGE_TYPE", "postgres") +@patch("iss_health_check_environment.ISS_KEY_TABLE_NAME", "test-table-name") @patch("addons.images.iss_health_check.iss_token.requests.Response") @patch("addons.images.iss_health_check.iss_token.requests") @patch("addons.images.iss_health_check.iss_token.uuid") diff --git a/services/addons/tests/obu_ota_server/conftest.py b/services/addons/tests/obu_ota_server/conftest.py new file mode 100644 index 000000000..80e8561cf --- /dev/null +++ b/services/addons/tests/obu_ota_server/conftest.py @@ -0,0 +1,3 @@ +import os + +os.environ['OTA_PASSWORD'] = 'ota-password' diff --git a/services/addons/tests/obu_ota_server/test_obu_ota_server.py b/services/addons/tests/obu_ota_server/test_obu_ota_server.py index 675977d16..75deca05b 100644 --- a/services/addons/tests/obu_ota_server/test_obu_ota_server.py +++ b/services/addons/tests/obu_ota_server/test_obu_ota_server.py @@ -17,23 +17,21 @@ ) -@patch("os.getenv") +@patch("obu_ota_server_environment.BLOB_STORAGE_PROVIDER", "DOCKER") @patch("glob.glob") -def test_get_firmware_list_local(mock_glob, mock_getenv): - mock_getenv.return_value = "DOCKER" +def test_get_firmware_list_local(mock_glob): mock_glob.return_value = ["/firmwares/test1.tar.sig", "/firmwares/test2.tar.sig"] result = get_firmware_list() - mock_getenv.assert_called_once_with("BLOB_STORAGE_PROVIDER", "DOCKER") mock_glob.assert_called_once_with("/firmwares/*.tar.sig") assert result == ["/firmwares/test1.tar.sig", "/firmwares/test2.tar.sig"] -@patch("os.getenv") +@patch("obu_ota_server_environment.BLOB_STORAGE_PROVIDER", "GCP") +@patch("obu_ota_server_environment.BLOB_STORAGE_PATH", "PATH") @patch("common.gcs_utils.list_gcs_blobs") -def test_get_firmware_list_gcs(mock_list_gcs_blobs, mock_getenv): - mock_getenv.return_value = "GCP" +def test_get_firmware_list_gcs(mock_list_gcs_blobs): mock_list_gcs_blobs.return_value = [ "/firmwares/test1.tar.sig", "/firmwares/test2.tar.sig", @@ -41,56 +39,46 @@ def test_get_firmware_list_gcs(mock_list_gcs_blobs, mock_getenv): result = get_firmware_list() - # mock_getenv.assert_called_once_with("BLOB_STORAGE_PROVIDER", "DOCKER") - mock_list_gcs_blobs.assert_called_once_with("GCP", ".tar.sig") + mock_list_gcs_blobs.assert_called_once_with("PATH", ".tar.sig") assert result == ["/firmwares/test1.tar.sig", "/firmwares/test2.tar.sig"] -@patch("os.getenv") +@patch("obu_ota_server_environment.BLOB_STORAGE_PROVIDER", "DOCKER") @patch("os.path.exists") @patch("common.gcs_utils.list_gcs_blobs") -def test_get_firmware_local_fail(mock_gcs_utils, mock_os_path_exists, mock_os_getenv): - mock_os_getenv.return_value = "DOCKER" +def test_get_firmware_local_fail(mock_gcs_utils, mock_os_path_exists): mock_os_path_exists.return_value = False firmware_id = "test_firmware_id" local_file_path = "test_local_file_path" result = get_firmware(firmware_id, local_file_path) - mock_os_getenv.assert_called_once_with("BLOB_STORAGE_PROVIDER", "DOCKER") mock_os_path_exists.assert_called_once_with(local_file_path) mock_gcs_utils.assert_not_called() - assert result == False + assert result is False -@patch("os.getenv") +@patch("obu_ota_server_environment.BLOB_STORAGE_PROVIDER", "DOCKER") @patch("os.path.exists") @patch("common.gcs_utils.list_gcs_blobs") -def test_get_firmware_local_success( - mock_gcs_utils, mock_os_path_exists, mock_os_getenv -): - mock_os_getenv.return_value = "DOCKER" +def test_get_firmware_local_success(mock_gcs_utils, mock_os_path_exists): mock_os_path_exists.return_value = True firmware_id = "test_firmware_id" local_file_path = "test_local_file_path" result = get_firmware(firmware_id, local_file_path) - mock_os_getenv.assert_called_once_with("BLOB_STORAGE_PROVIDER", "DOCKER") mock_os_path_exists.assert_called_once_with(local_file_path) mock_gcs_utils.assert_not_called() - assert result == True + assert result is True -@patch("os.getenv") +@patch("obu_ota_server_environment.BLOB_STORAGE_PROVIDER", "GCP") @patch("os.path.exists") @patch("common.gcs_utils.download_gcp_blob") -def test_get_firmware_gcs_success( - mock_download_gcp_blob, mock_os_path_exists, mock_os_getenv -): - mock_os_getenv.return_value = "GCP" +def test_get_firmware_gcs_success(mock_download_gcp_blob, mock_os_path_exists): mock_os_path_exists.return_value = False mock_download_gcp_blob.return_value = True @@ -99,22 +87,18 @@ def test_get_firmware_gcs_success( local_file_path = "test_local_file_path" result = get_firmware(firmware_id, local_file_path) - mock_os_getenv.assert_called_with("BLOB_STORAGE_PROVIDER", "DOCKER") mock_os_path_exists.assert_called_with(local_file_path) mock_download_gcp_blob.assert_called_once_with( firmware_id, local_file_path, firmware_file_ext ) - assert result == True + assert result is True -@patch("os.getenv") +@patch("obu_ota_server_environment.BLOB_STORAGE_PROVIDER", "GCP") @patch("os.path.exists") @patch("common.gcs_utils.download_gcp_blob") -def test_get_firmware_gcs_failure( - mock_download_gcp_blob, mock_os_path_exists, mock_os_getenv -): - mock_os_getenv.return_value = "GCP" +def test_get_firmware_gcs_failure(mock_download_gcp_blob, mock_os_path_exists): mock_os_path_exists.return_value = False mock_download_gcp_blob.return_value = False @@ -123,13 +107,12 @@ def test_get_firmware_gcs_failure( local_file_path = "test_local_file_path" result = get_firmware(firmware_id, local_file_path) - mock_os_getenv.assert_called_with("BLOB_STORAGE_PROVIDER", "DOCKER") mock_os_path_exists.assert_called_with(local_file_path) mock_download_gcp_blob.assert_called_once_with( firmware_id, local_file_path, firmware_file_ext ) - assert result == False + assert result is False def test_parse_range_header_valid(): @@ -198,7 +181,14 @@ async def test_read_file_no_end_range(): os.remove(temp_path) -@patch.dict("os.environ", {"OTA_USERNAME": "username", "OTA_PASSWORD": "password"}) +@patch( + "obu_ota_server_environment.OTA_USERNAME", + "username", +) +@patch( + "obu_ota_server_environment.OTA_PASSWORD", + "password", +) @pytest.mark.anyio async def test_read_root(): async with AsyncClient( @@ -209,10 +199,17 @@ async def test_read_root(): assert response.json() == {"message": "obu ota server healthcheck", "root_path": ""} -@patch.dict("os.environ", {"OTA_USERNAME": "username", "OTA_PASSWORD": "password"}) -@pytest.mark.anyio +@patch( + "obu_ota_server_environment.OTA_USERNAME", + "username", +) +@patch( + "obu_ota_server_environment.OTA_PASSWORD", + "password", +) @patch("addons.images.obu_ota_server.obu_ota_server.get_firmware_list") @patch("addons.images.obu_ota_server.obu_ota_server.commsignia_manifest.add_contents") +@pytest.mark.anyio async def test_get_manifest(mock_commsignia_manifest, mock_get_firmware_list): mock_get_firmware_list.return_value = [ "/firmwares/test1.tar.sig", @@ -229,11 +226,18 @@ async def test_get_manifest(mock_commsignia_manifest, mock_get_firmware_list): assert response.json() == {"json": "data"} -@patch.dict("os.environ", {"OTA_USERNAME": "username", "OTA_PASSWORD": "password"}) -@pytest.mark.anyio +@patch( + "obu_ota_server_environment.OTA_USERNAME", + "username", +) +@patch( + "obu_ota_server_environment.OTA_PASSWORD", + "password", +) @patch("addons.images.obu_ota_server.obu_ota_server.get_firmware") @patch("addons.images.obu_ota_server.obu_ota_server.parse_range_header") @patch("addons.images.obu_ota_server.obu_ota_server.read_file") +@pytest.mark.anyio async def test_get_fw(mock_read_file, mock_parse_range_header, mock_get_firmware): mock_get_firmware.return_value = True mock_parse_range_header.return_value = 0, 100 @@ -250,10 +254,10 @@ async def test_get_fw(mock_read_file, mock_parse_range_header, mock_get_firmware assert response.headers["Content-Length"] == "100" -@pytest.mark.asyncio @patch("addons.images.obu_ota_server.obu_ota_server.pgquery") @patch("addons.images.obu_ota_server.obu_ota_server.datetime") @patch("addons.images.obu_ota_server.obu_ota_server.removed_old_logs") +@pytest.mark.asyncio async def test_log_request(mock_removed_old_logs, mock_datetime, mock_pgquery): fixed_datetime = datetime(2024, 7, 30, 0, 0, 0) mock_datetime.now.return_value = fixed_datetime @@ -291,7 +295,7 @@ async def test_log_request(mock_removed_old_logs, mock_datetime, mock_pgquery): ) -@patch.dict("os.environ", {"MAX_COUNT": "10"}) +@patch("obu_ota_server_environment.MAX_COUNT", 10) @patch("addons.images.obu_ota_server.obu_ota_server.pgquery") def test_removed_old_logs_no_removal(mock_pgquery): mock_pgquery.query_db.side_effect = [ @@ -307,7 +311,7 @@ def test_removed_old_logs_no_removal(mock_pgquery): mock_pgquery.write_db.assert_not_called() -@patch.dict("os.environ", {"MAX_COUNT": "5"}) +@patch("obu_ota_server_environment.MAX_COUNT", 5) @patch("addons.images.obu_ota_server.obu_ota_server.pgquery") def test_removed_old_logs_with_removal(mock_pgquery): mock_pgquery.query_db.side_effect = [ @@ -330,10 +334,17 @@ def test_removed_old_logs_with_removal(mock_pgquery): ) -@patch.dict("os.environ", {"OTA_USERNAME": "username", "OTA_PASSWORD": "password"}) -@pytest.mark.anyio +@patch( + "obu_ota_server_environment.OTA_USERNAME", + "username", +) +@patch( + "obu_ota_server_environment.OTA_PASSWORD", + "password", +) @patch("addons.images.obu_ota_server.obu_ota_server.get_firmware_list") @patch("addons.images.obu_ota_server.obu_ota_server.commsignia_manifest.add_contents") +@pytest.mark.anyio async def test_get_manifest(mock_commsignia_manifest, mock_get_firmware_list): mock_get_firmware_list.return_value = [ "/firmwares/test1.tar.sig", @@ -350,17 +361,12 @@ async def test_get_manifest(mock_commsignia_manifest, mock_get_firmware_list): assert response.json() == {"json": "data"} -@patch.dict( - "os.environ", - { - "OTA_USERNAME": "username", - "OTA_PASSWORD": "password", - "NGINX_ENCRYPTION": "plain", - }, -) -@pytest.mark.anyio +@patch("obu_ota_server_environment.OTA_USERNAME", "username") +@patch("obu_ota_server_environment.OTA_PASSWORD", "password") +@patch("obu_ota_server_environment.NGINX_ENCRYPTION", "plain") @patch("addons.images.obu_ota_server.obu_ota_server.get_firmware_list") @patch("addons.images.obu_ota_server.obu_ota_server.commsignia_manifest.add_contents") +@pytest.mark.anyio async def test_fqdn_response_plain(mock_commsignia_manifest, mock_get_firmware_list): mock_get_firmware_list.return_value = [] expected_hostname = "http://localhost" @@ -377,17 +383,12 @@ async def test_fqdn_response_plain(mock_commsignia_manifest, mock_get_firmware_l mock_commsignia_manifest.assert_called_once_with(expected_hostname, []) -@patch.dict( - "os.environ", - { - "OTA_USERNAME": "username", - "OTA_PASSWORD": "password", - "NGINX_ENCRYPTION": "SSL", - }, -) -@pytest.mark.anyio +@patch("obu_ota_server_environment.OTA_USERNAME", "username") +@patch("obu_ota_server_environment.OTA_PASSWORD", "password") +@patch("obu_ota_server_environment.NGINX_ENCRYPTION", "ssl") @patch("addons.images.obu_ota_server.obu_ota_server.get_firmware_list") @patch("addons.images.obu_ota_server.obu_ota_server.commsignia_manifest.add_contents") +@pytest.mark.anyio async def test_fqdn_response_ssl(mock_commsignia_manifest, mock_get_firmware_list): mock_get_firmware_list.return_value = [] expected_hostname = "https://localhost" diff --git a/services/addons/tests/rsu_status_check/conftest.py b/services/addons/tests/rsu_status_check/conftest.py new file mode 100644 index 000000000..25727936d --- /dev/null +++ b/services/addons/tests/rsu_status_check/conftest.py @@ -0,0 +1,5 @@ +import os + +os.environ['ZABBIX_ENDPOINT'] = 'zabbix-endpoint' +os.environ['ZABBIX_USER'] = 'zabbix-user' +os.environ['ZABBIX_PASSWORD'] = 'zabbix-password' diff --git a/services/addons/tests/rsu_status_check/test_rsu_msgfwd_fetch.py b/services/addons/tests/rsu_status_check/test_rsu_msgfwd_fetch.py index aeba21915..c9ca516f8 100644 --- a/services/addons/tests/rsu_status_check/test_rsu_msgfwd_fetch.py +++ b/services/addons/tests/rsu_status_check/test_rsu_msgfwd_fetch.py @@ -2,7 +2,7 @@ from addons.images.rsu_status_check.rsu_msgfwd_fetch import main -@patch("os.environ", {"RSU_MSGFWD_FETCH": "False"}) +@patch("rsu_status_check_environment.RSU_MSGFWD_FETCH", False) @patch( "addons.images.rsu_status_check.rsu_msgfwd_fetch.UpdatePostgresRsuMessageForward" ) @@ -15,7 +15,7 @@ def test_main_service_disabled(mock_logging, mock_update_pg): mock_update_pg.assert_not_called() -@patch("os.environ", {"RSU_MSGFWD_FETCH": "True"}) +@patch("rsu_status_check_environment.RSU_MSGFWD_FETCH", True) @patch( "addons.images.rsu_status_check.rsu_msgfwd_fetch.UpdatePostgresRsuMessageForward" ) diff --git a/services/addons/tests/rsu_status_check/test_rsu_ping_fetch.py b/services/addons/tests/rsu_status_check/test_rsu_ping_fetch.py index 7c0b0880a..499ec7278 100644 --- a/services/addons/tests/rsu_status_check/test_rsu_ping_fetch.py +++ b/services/addons/tests/rsu_status_check/test_rsu_ping_fetch.py @@ -78,12 +78,12 @@ def test_insert_rsu_ping(mock_write_db): def createRsuStatusFetchInstance(): - rsu_ping_fetch.os.environ["ZABBIX_ENDPOINT"] = "endpoint" - rsu_ping_fetch.os.environ["ZABBIX_USER"] = "user" - rsu_ping_fetch.os.environ["ZABBIX_PASSWORD"] = "password" return rsu_ping_fetch.RsuStatusFetch() +@patch("rsu_status_check_environment.ZABBIX_ENDPOINT", "endpoint") +@patch("rsu_status_check_environment.ZABBIX_USER", "user") +@patch("rsu_status_check_environment.ZABBIX_PASSWORD", "password") def test_setZabbixAuth(): # prepare rsf = createRsuStatusFetchInstance() @@ -110,6 +110,9 @@ def test_setZabbixAuth(): assert rsf.ZABBIX_AUTH == "auth" +@patch("rsu_status_check_environment.ZABBIX_ENDPOINT", "endpoint") +@patch("rsu_status_check_environment.ZABBIX_USER", "user") +@patch("rsu_status_check_environment.ZABBIX_PASSWORD", "password") def test_getHostInfo(): # prepare rsf = createRsuStatusFetchInstance() @@ -139,6 +142,9 @@ def test_getHostInfo(): assert result == {"result": "result"} +@patch("rsu_status_check_environment.ZABBIX_ENDPOINT", "endpoint") +@patch("rsu_status_check_environment.ZABBIX_USER", "user") +@patch("rsu_status_check_environment.ZABBIX_PASSWORD", "password") def test_getItem(): # prepare rsf = createRsuStatusFetchInstance() @@ -170,6 +176,9 @@ def test_getItem(): assert result == {"result": "result"} +@patch("rsu_status_check_environment.ZABBIX_ENDPOINT", "endpoint") +@patch("rsu_status_check_environment.ZABBIX_USER", "user") +@patch("rsu_status_check_environment.ZABBIX_PASSWORD", "password") def test_getHistory(): # prepare rsf = createRsuStatusFetchInstance() @@ -207,6 +216,9 @@ def test_getHistory(): assert result == {"result": "result"} +@patch("rsu_status_check_environment.ZABBIX_ENDPOINT", "endpoint") +@patch("rsu_status_check_environment.ZABBIX_USER", "user") +@patch("rsu_status_check_environment.ZABBIX_PASSWORD", "password") def test_insertHistoryItem(): # prepare rsf = createRsuStatusFetchInstance() @@ -236,6 +248,9 @@ def test_insertHistoryItem(): assert result == True +@patch("rsu_status_check_environment.ZABBIX_ENDPOINT", "endpoint") +@patch("rsu_status_check_environment.ZABBIX_USER", "user") +@patch("rsu_status_check_environment.ZABBIX_PASSWORD", "password") def test_printConfigInfo(): # prepare rsf = createRsuStatusFetchInstance() @@ -250,6 +265,9 @@ def test_printConfigInfo(): rsu_ping_fetch.logging.info.assert_called_once_with(expected_message) +@patch("rsu_status_check_environment.ZABBIX_ENDPOINT", "endpoint") +@patch("rsu_status_check_environment.ZABBIX_USER", "user") +@patch("rsu_status_check_environment.ZABBIX_PASSWORD", "password") def test_run(): # prepare rsf = createRsuStatusFetchInstance() @@ -281,6 +299,9 @@ def test_run(): rsu_ping_fetch.logging.error.assert_not_called() +@patch("rsu_status_check_environment.ZABBIX_ENDPOINT", "endpoint") +@patch("rsu_status_check_environment.ZABBIX_USER", "user") +@patch("rsu_status_check_environment.ZABBIX_PASSWORD", "password") def test_run_insert_failure(): # prepare rsf = createRsuStatusFetchInstance() @@ -314,6 +335,9 @@ def test_run_insert_failure(): rsu_ping_fetch.logging.error.assert_not_called() +@patch("rsu_status_check_environment.ZABBIX_ENDPOINT", "endpoint") +@patch("rsu_status_check_environment.ZABBIX_USER", "user") +@patch("rsu_status_check_environment.ZABBIX_PASSWORD", "password") def test_run_exception(): # prepare rsf = createRsuStatusFetchInstance() diff --git a/services/api/README.md b/services/api/README.md index ac963eddd..a865cd2a9 100644 --- a/services/api/README.md +++ b/services/api/README.md @@ -335,11 +335,10 @@ HTTP URL Arguments: - PG_DB_PORT: The database port. - PG_PG_DB_USER: The database user that will be used to authenticate the cloud function when it queries the database. - PG_PG_DB_PASS: The database user's password that will be used to authenticate the cloud function. -- COUNTS_MSG_TYPES: Set to a list of message types to include in counts query. Sample format is described in the sample.env. - MONGO_PROCESSED_BSM_COLLECTION_NAME: The database name for processed BSM messages output from the [Geojson Converter](https://github.com/usdot-jpo-ode/geojson-converter). - MONGO_PROCESSED_PSM_COLLECTION_NAME: The database name for processed PSM messages output from the [Geojson Converter](https://github.com/usdot-jpo-ode/geojson-converter). -- SSM_DB_NAME: The database name for SSM visualization data. -- SRM_DB_NAME: The database name for SRM visualization data. +- MONGO_SSM_COLLECTION_NAME: The database name for SSM visualization data. +- MONGO_SRM_COLLECTION_NAME: The database name for SRM visualization data. - MONGO_DB_URI: URI for the MongoDB connection. - MONGO_DB_NAME: Database name for RSU counts. - KEYCLOAK_ENDPOINT: Keycloak base URL to send requests to. Reference the sample.env for the URL formatting. diff --git a/services/api/requirements.txt b/services/api/requirements.txt index 1a8710b78..567e90c8f 100644 --- a/services/api/requirements.txt +++ b/services/api/requirements.txt @@ -10,7 +10,7 @@ DateTime==5.2 python-dateutil==2.8.2 pytz==2023.3.post1 Werkzeug==3.0.0 -python-keycloak==5.5.1 +python-keycloak==5.8.1 pymongo==4.5.0 fabric==3.2.2 google-cloud-bigquery==3.29.0 diff --git a/services/api/sample.env b/services/api/sample.env index d56235702..294d7f8c1 100644 --- a/services/api/sample.env +++ b/services/api/sample.env @@ -3,49 +3,54 @@ # If using docker then this value should be set to: http://${WEBAPP_HOST_IP}:3000 # If running the webapp using npm then set it to: http://localhost:3000 # Leave as * to allow all domains access -CORS_DOMAIN = * +CORS_DOMAIN=* # PostgreSQL Database connection information # this value may need to folow with the webapp host if debugging the applications -PG_DB_HOST=:5432 -PG_DB_NAME= -PG_DB_USER= +PG_DB_HOST=localhost:5432 +PG_DB_NAME=postgres +PG_DB_USER=postgres # If the PG_DB_PASS variable has special characters, make sure to wrap it in single quotes -PG_DB_PASS= +PG_DB_PASS=postgres # If connecting to PGDB over websocket: INSTANCE_CONNECTION_NAME= # Keycloak Variables -KEYCLOAK_ENDPOINT= http://cvmanager.auth.com:8084/ -KEYCLOAK_REALM= -KEYCLOAK_API_CLIENT_ID= -KEYCLOAK_API_CLIENT_SECRET_KEY= +KEYCLOAK_ENDPOINT=http://localhost:8084 +KEYCLOAK_REALM=cvmanager +KEYCLOAK_API_CLIENT_ID=cvmanager-api +KEYCLOAK_API_CLIENT_SECRET_KEY=keycloak-secret-key # Firmware Manager connectivity in the format 'http://endpoint:port' -FIRMWARE_MANAGER_ENDPOINT=http://:8089 +FIRMWARE_MANAGER_ENDPOINT=http://localhost:8089 # If "BIGQUERY", set the location of the GCP service account key GOOGLE_APPLICATION_CREDENTIALS='./resources/google/sample_gcp_service_account.json' # If "MONGODB", MongoDB variables -MONGO_DB_URI= -MONGO_DB_NAME="ODE" +MONGO_DB_URI=mongodb://ode:replace-me@localhost:27017/?directConnection=true&authSource=admin +MONGO_DB_NAME="CV" # Set these variables if using either "MONGODB" or "BIGQUERY" -# COUNTS_MSG_TYPES: Comma seperated list of message types -COUNTS_MSG_TYPES='BSM,SSM,SPAT,SRM,MAP' -MONGO_PROCESSED_BSM_COLLECTION_NAME="ProcessedBsm" -MONGO_PROCESSED_PSM_COLLECTION_NAME="ProcessedPsm" -SSM_DB_NAME= -SRM_DB_NAME= +MONGO_PROCESSED_BSM_COLLECTION_NAME="processed_bsm" +MONGO_PROCESSED_PSM_COLLECTION_NAME="processed_psm" +MONGO_SSM_COLLECTION_NAME= +MONGO_SRM_COLLECTION_NAME= # Specifies the maximum number of V2x messages returned from the geo_query_geo_data_mongo method before filtering occurs MAX_GEO_QUERY_RECORDS= +# Feature Flags +# Disable WZDx without WZDx Endpoint +ENABLE_RSU_FEATURES=true +ENABLE_INTERSECTION_FEATURES=true +ENABLE_WZDX_FEATURES=false +ENABLE_MOOVE_AI_FEATURES=false + # WZDX Variables -WZDX_API_KEY = -WZDX_ENDPOINT = +WZDX_API_KEY= +WZDX_ENDPOINT= # Contact Support Menu Email Configuration CSM_EMAIL_TO_SEND_FROM= diff --git a/services/api/src/admin_email_notification.py b/services/api/src/admin_email_notification.py index c7640efa7..67f69e2aa 100644 --- a/services/api/src/admin_email_notification.py +++ b/services/api/src/admin_email_notification.py @@ -4,8 +4,8 @@ import urllib.request import logging import common.pgquery as pgquery +import api_environment from sqlalchemy.exc import IntegrityError, SQLAlchemyError -import os from werkzeug.exceptions import InternalServerError, BadRequest from common.auth_tools import ( @@ -157,14 +157,14 @@ class AdminNotificationPatchSchema(Schema): class AdminNotification(Resource): options_headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Access-Control-Allow-Headers": "Content-Type,Authorization", "Access-Control-Allow-Methods": "GET,PATCH,DELETE", "Access-Control-Max-Age": "3600", } headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Content-Type": "application/json", } diff --git a/services/api/src/admin_intersection.py b/services/api/src/admin_intersection.py index 9e832cd75..5e7611e39 100644 --- a/services/api/src/admin_intersection.py +++ b/services/api/src/admin_intersection.py @@ -6,7 +6,7 @@ import common.pgquery as pgquery from sqlalchemy.exc import IntegrityError, SQLAlchemyError import admin_new_intersection -import os +import api_environment from werkzeug.exceptions import InternalServerError, BadRequest from common.auth_tools import ( ORG_ROLE_LITERAL, @@ -57,7 +57,12 @@ def get_intersection_data( where_clauses = [] params: dict[str, Any] = {} - if not user.user_info.super_user: + + # Filter by user's organization + if user.organization is not None: + where_clauses.append("org.name = :user_org_name") + params["user_org_name"] = user.organization + elif not user.user_info.super_user: org_names_placeholder, _ = generate_sql_placeholders_for_list( qualified_orgs, params_to_update=params ) @@ -350,14 +355,14 @@ class AdminIntersectionPatchSchema(Schema): class AdminIntersection(Resource): options_headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], - "Access-Control-Allow-Headers": "Content-Type,Authorization", + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, + "Access-Control-Allow-Headers": "Content-Type,Authorization,Organization", "Access-Control-Allow-Methods": "GET,PATCH,DELETE", "Access-Control-Max-Age": "3600", } headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Content-Type": "application/json", } diff --git a/services/api/src/admin_new_email_notification.py b/services/api/src/admin_new_email_notification.py index 51ac1f35c..483c3a24c 100644 --- a/services/api/src/admin_new_email_notification.py +++ b/services/api/src/admin_new_email_notification.py @@ -3,8 +3,8 @@ from marshmallow import Schema, fields import logging import common.pgquery as pgquery +import api_environment from sqlalchemy.exc import IntegrityError, SQLAlchemyError -import os from werkzeug.exceptions import InternalServerError, BadRequest from common.auth_tools import ( ORG_ROLE_LITERAL, @@ -99,14 +99,14 @@ class AdminGetNotificationSchema(Schema): class AdminNewNotification(Resource): options_headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Access-Control-Allow-Headers": "Content-Type,Authorization", "Access-Control-Allow-Methods": "GET,POST", "Access-Control-Max-Age": "3600", } headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Content-Type": "application/json", } diff --git a/services/api/src/admin_new_intersection.py b/services/api/src/admin_new_intersection.py index fc4bcb2f4..fbf7598d4 100644 --- a/services/api/src/admin_new_intersection.py +++ b/services/api/src/admin_new_intersection.py @@ -3,8 +3,8 @@ from marshmallow import Schema, fields, validate import logging import common.pgquery as pgquery +import api_environment from sqlalchemy.exc import IntegrityError, SQLAlchemyError -import os from werkzeug.exceptions import InternalServerError, BadRequest from common.auth_tools import ( ORG_ROLE_LITERAL, @@ -242,14 +242,14 @@ class AdminNewIntersectionSchema(Schema): class AdminNewIntersection(Resource): options_headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Access-Control-Allow-Headers": "Content-Type,Authorization", "Access-Control-Allow-Methods": "GET,POST", "Access-Control-Max-Age": "3600", } headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Content-Type": "application/json", } diff --git a/services/api/src/admin_new_org.py b/services/api/src/admin_new_org.py index f123c814d..37786f1e1 100644 --- a/services/api/src/admin_new_org.py +++ b/services/api/src/admin_new_org.py @@ -3,8 +3,8 @@ from marshmallow import Schema, fields import logging import common.pgquery as pgquery +import api_environment from sqlalchemy.exc import IntegrityError, SQLAlchemyError -import os import admin_new_user from werkzeug.exceptions import InternalServerError, BadRequest from common.auth_tools import require_permission @@ -61,14 +61,14 @@ class AdminNewOrgSchema(Schema): class AdminNewOrg(Resource): options_headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Access-Control-Allow-Headers": "Content-Type,Authorization", "Access-Control-Allow-Methods": "POST", "Access-Control-Max-Age": "3600", } headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Content-Type": "application/json", } diff --git a/services/api/src/admin_new_rsu.py b/services/api/src/admin_new_rsu.py index cb6548721..e228d95b0 100644 --- a/services/api/src/admin_new_rsu.py +++ b/services/api/src/admin_new_rsu.py @@ -4,8 +4,8 @@ from marshmallow import Schema, fields, validate import logging import common.pgquery as pgquery +import api_environment from sqlalchemy.exc import IntegrityError, SQLAlchemyError -import os from werkzeug.exceptions import InternalServerError, BadRequest from common.auth_tools import ( ORG_ROLE_LITERAL, @@ -184,14 +184,14 @@ class AdminNewRsuSchema(Schema): class AdminNewRsu(Resource): options_headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Access-Control-Allow-Headers": "Content-Type,Authorization", "Access-Control-Allow-Methods": "GET,POST", "Access-Control-Max-Age": "3600", } headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Content-Type": "application/json", } diff --git a/services/api/src/admin_new_user.py b/services/api/src/admin_new_user.py index 565c2f42c..0cb89e553 100644 --- a/services/api/src/admin_new_user.py +++ b/services/api/src/admin_new_user.py @@ -4,8 +4,8 @@ from marshmallow import Schema, fields, validate import logging import common.pgquery as pgquery +import api_environment from sqlalchemy.exc import IntegrityError, SQLAlchemyError -import os import time from werkzeug.exceptions import InternalServerError, BadRequest from common.auth_tools import ( @@ -150,14 +150,14 @@ class AdminNewUserSchema(Schema): class AdminNewUser(Resource): options_headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Access-Control-Allow-Headers": "Content-Type,Authorization", "Access-Control-Allow-Methods": "GET,POST", "Access-Control-Max-Age": "3600", } headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Content-Type": "application/json", } diff --git a/services/api/src/admin_org.py b/services/api/src/admin_org.py index 809cc4137..d116ce3c5 100644 --- a/services/api/src/admin_org.py +++ b/services/api/src/admin_org.py @@ -7,7 +7,7 @@ import common.pgquery as pgquery from sqlalchemy.exc import IntegrityError, SQLAlchemyError import admin_new_user -import os +import api_environment from werkzeug.exceptions import InternalServerError, BadRequest, Conflict, Forbidden from common.auth_tools import ( ORG_ROLE_LITERAL, @@ -505,14 +505,14 @@ class AdminOrgPatchSchema(Schema): class AdminOrg(Resource): options_headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Access-Control-Allow-Headers": "Content-Type,Authorization", "Access-Control-Allow-Methods": "GET,PATCH,DELETE", "Access-Control-Max-Age": "3600", } headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Content-Type": "application/json", } diff --git a/services/api/src/admin_rsu.py b/services/api/src/admin_rsu.py index 0d56358cb..e5266b9b0 100644 --- a/services/api/src/admin_rsu.py +++ b/services/api/src/admin_rsu.py @@ -6,7 +6,7 @@ import common.pgquery as pgquery from sqlalchemy.exc import IntegrityError, SQLAlchemyError import admin_new_rsu -import os +import api_environment from werkzeug.exceptions import InternalServerError, BadRequest from common.auth_tools import ( ORG_ROLE_LITERAL, @@ -38,7 +38,12 @@ def get_rsu_data(rsu_ip: str, user: EnvironWithOrg, qualified_orgs: list[str]): where_clauses = [] params: dict[str, Any] = {} - if not user.user_info.super_user: + + # Filter by user's organization + if user.organization: + where_clauses.append("org.name = :user_org_name") + params["user_org_name"] = user.organization + elif not user.user_info.super_user: org_names_placeholder, _ = generate_sql_placeholders_for_list( qualified_orgs, params_to_update=params ) @@ -283,14 +288,14 @@ class AdminRsuPatchSchema(Schema): class AdminRsu(Resource): options_headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], - "Access-Control-Allow-Headers": "Content-Type,Authorization", + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, + "Access-Control-Allow-Headers": "Content-Type,Authorization,Organization", "Access-Control-Allow-Methods": "GET,PATCH,DELETE", "Access-Control-Max-Age": "3600", } headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Content-Type": "application/json", } diff --git a/services/api/src/admin_user.py b/services/api/src/admin_user.py index 1875b43b7..ab324b9c4 100644 --- a/services/api/src/admin_user.py +++ b/services/api/src/admin_user.py @@ -7,7 +7,7 @@ import common.pgquery as pgquery from sqlalchemy.exc import IntegrityError, SQLAlchemyError import admin_new_user -import os +import api_environment from werkzeug.exceptions import InternalServerError, BadRequest from common.auth_tools import ( ORG_ROLE_LITERAL, @@ -33,7 +33,12 @@ def get_user_data(user_email: str, user: EnvironWithOrg, qualified_orgs: list[st where_clauses = [] params: dict[str, Any] = {} - if not user.user_info.super_user: + + # Filter by user's organization + if user.organization: + where_clauses.append("org.name = :user_org_name") + params["user_org_name"] = user.organization + elif not user.user_info.super_user: org_names_placeholder, _ = generate_sql_placeholders_for_list( qualified_orgs, params_to_update=params ) @@ -271,14 +276,14 @@ class AdminUserPatchSchema(Schema): class AdminUser(Resource): options_headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], - "Access-Control-Allow-Headers": "Content-Type,Authorization", + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, + "Access-Control-Allow-Headers": "Content-Type,Authorization,Organization", "Access-Control-Allow-Methods": "GET,PATCH,DELETE", "Access-Control-Max-Age": "3600", } headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Content-Type": "application/json", } diff --git a/services/api/src/api_environment.py b/services/api/src/api_environment.py new file mode 100644 index 000000000..ed7f66329 --- /dev/null +++ b/services/api/src/api_environment.py @@ -0,0 +1,58 @@ +from common.common_environment import get_env_var + + +APPLICATION_PORT = int(get_env_var("FLASK_RUN_PORT", "5000")) + +ENABLE_RSU_FEATURES = get_env_var("ENABLE_RSU_FEATURES", "true").lower() != "false" +ENABLE_INTERSECTION_FEATURES = ( + get_env_var("ENABLE_INTERSECTION_FEATURES", "true").lower() != "false" +) +ENABLE_WZDX_FEATURES = get_env_var("ENABLE_WZDX_FEATURES", "true").lower() != "false" +ENABLE_MOOVE_AI_FEATURES = ( + get_env_var("ENABLE_MOOVE_AI_FEATURES", "true").lower() != "false" +) + +KEYCLOAK_ENDPOINT = get_env_var("KEYCLOAK_ENDPOINT", error=True) +KEYCLOAK_REALM = get_env_var("KEYCLOAK_REALM", error=True) +KEYCLOAK_API_CLIENT_ID = get_env_var("KEYCLOAK_API_CLIENT_ID", error=True) +KEYCLOAK_API_CLIENT_SECRET_KEY = get_env_var( + "KEYCLOAK_API_CLIENT_SECRET_KEY", error=True +) + +CORS_DOMAIN = get_env_var("CORS_DOMAIN", "*") +CSM_EMAILS_TO_SEND_TO = get_env_var("CSM_EMAILS_TO_SEND_TO", "").split(",") +CSM_EMAIL_TO_SEND_FROM = get_env_var("CSM_EMAIL_TO_SEND_FROM") +CSM_TARGET_SMTP_SERVER_ADDRESS = get_env_var("CSM_TARGET_SMTP_SERVER_ADDRESS") +CSM_TARGET_SMTP_SERVER_PORT = int(get_env_var("CSM_TARGET_SMTP_SERVER_PORT", "587")) +CSM_TLS_ENABLED = get_env_var("CSM_TLS_ENABLED", "true").lower() == "true" +CSM_AUTH_ENABLED = get_env_var("CSM_AUTH_ENABLED", "true").lower() == "true" +CSM_EMAIL_APP_USERNAME = get_env_var("CSM_EMAIL_APP_USERNAME", warn=CSM_AUTH_ENABLED) +CSM_EMAIL_APP_PASSWORD = get_env_var("CSM_EMAIL_APP_PASSWORD", warn=CSM_AUTH_ENABLED) + +MONGO_DB_URI = get_env_var("MONGO_DB_URI", "mongodb://localhost:27017/", warn=True) +MONGO_DB_NAME = get_env_var("MONGO_DB_NAME", "CV", warn=True) +MONGO_SSM_COLLECTION_NAME = get_env_var("MONGO_SSM_COLLECTION_NAME") +MONGO_SRM_COLLECTION_NAME = get_env_var("MONGO_SRM_COLLECTION_NAME") +MONGO_PROCESSED_BSM_COLLECTION_NAME = get_env_var( + "MONGO_PROCESSED_BSM_COLLECTION_NAME", "ProcessedBsm", warn=False +) +MONGO_PROCESSED_PSM_COLLECTION_NAME = get_env_var( + "MONGO_PROCESSED_PSM_COLLECTION_NAME", "ProcessedPsm", warn=False +) +MAX_GEO_QUERY_RECORDS = int(get_env_var("MAX_GEO_QUERY_RECORDS", "10000", warn=False)) + +ENVIRONMENT_NAME = get_env_var("ENVIRONMENT_NAME") +LOGS_LINK = get_env_var("LOGS_LINK") + +WZDX_ENDPOINT = get_env_var("WZDX_ENDPOINT", error=ENABLE_WZDX_FEATURES) +WZDX_API_KEY = get_env_var("WZDX_API_KEY", error=ENABLE_WZDX_FEATURES) + +FIRMWARE_MANAGER_ENDPOINT = get_env_var("FIRMWARE_MANAGER_ENDPOINT", warn=False) + +GCP_PROJECT_ID = get_env_var("GCP_PROJECT_ID", warn=ENABLE_MOOVE_AI_FEATURES) +MOOVE_AI_SEGMENT_AGG_STATS_TABLE = get_env_var( + "MOOVE_AI_SEGMENT_AGG_STATS_TABLE", warn=ENABLE_MOOVE_AI_FEATURES +) +MOOVE_AI_SEGMENT_EVENT_STATS_TABLE = get_env_var( + "MOOVE_AI_SEGMENT_EVENT_STATS_TABLE", warn=ENABLE_MOOVE_AI_FEATURES +) diff --git a/services/api/src/contact_support.py b/services/api/src/contact_support.py index decb785df..50bad6c8e 100644 --- a/services/api/src/contact_support.py +++ b/services/api/src/contact_support.py @@ -1,5 +1,6 @@ import logging -import os +import api_environment + from flask import abort, request from flask_restful import Resource from marshmallow import Schema @@ -17,31 +18,29 @@ class ContactSupportSchema(Schema): class ContactSupportResource(Resource): options_headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Access-Control-Allow-Headers": "Content-Type,Authorization", "Access-Control-Allow-Methods": "POST,OPTIONS", "Access-Control-Max-Age": "3600", } headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Access-Control-Allow-Headers": "Content-Type,Authorization", "Access-Control-Allow-Methods": "POST,OPTIONS", "Content-Type": "application/json", } def __init__(self): - self.CSM_EMAIL_TO_SEND_FROM = os.environ.get("CSM_EMAIL_TO_SEND_FROM") - self.CSM_TARGET_SMTP_SERVER_ADDRESS = os.environ.get( - "CSM_TARGET_SMTP_SERVER_ADDRESS" - ) - self.CSM_TARGET_SMTP_SERVER_PORT = int( - os.environ.get("CSM_TARGET_SMTP_SERVER_PORT") + self.CSM_EMAIL_TO_SEND_FROM = api_environment.CSM_EMAIL_TO_SEND_FROM + self.CSM_TARGET_SMTP_SERVER_ADDRESS = ( + api_environment.CSM_TARGET_SMTP_SERVER_ADDRESS ) - self.CSM_TLS_ENABLED = os.environ.get("CSM_TLS_ENABLED", "true") - self.CSM_AUTH_ENABLED = os.environ.get("CSM_AUTH_ENABLED", "true") - self.CSM_EMAIL_APP_USERNAME = os.environ.get("CSM_EMAIL_APP_USERNAME") - self.CSM_EMAIL_APP_PASSWORD = os.environ.get("CSM_EMAIL_APP_PASSWORD") + self.CSM_TARGET_SMTP_SERVER_PORT = api_environment.CSM_TARGET_SMTP_SERVER_PORT + self.CSM_TLS_ENABLED = api_environment.CSM_TLS_ENABLED + self.CSM_AUTH_ENABLED = api_environment.CSM_AUTH_ENABLED + self.CSM_EMAIL_APP_USERNAME = api_environment.CSM_EMAIL_APP_USERNAME + self.CSM_EMAIL_APP_PASSWORD = api_environment.CSM_EMAIL_APP_PASSWORD if not self.CSM_EMAIL_TO_SEND_FROM: logging.error("CSM_EMAIL_TO_SEND_FROM environment variable not set") @@ -53,7 +52,7 @@ def __init__(self): logging.error("CSM_TARGET_SMTP_SERVER_PORT environment variable not set") abort(500) - if self.CSM_AUTH_ENABLED == "true": + if self.CSM_AUTH_ENABLED: if not self.CSM_EMAIL_APP_USERNAME: logging.error("CSM_EMAIL_APP_USERNAME environment variable not set") abort(500) diff --git a/services/api/src/healthcheck.py b/services/api/src/healthcheck.py index ef50a0ab8..89bc1d6fa 100644 --- a/services/api/src/healthcheck.py +++ b/services/api/src/healthcheck.py @@ -1,4 +1,4 @@ -import os +import api_environment # REST endpoint resource class from flask_restful import Resource @@ -6,14 +6,14 @@ class HealthCheck(Resource): options_headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Access-Control-Allow-Headers": "Content-Type", "Access-Control-Allow-Methods": "GET", "Access-Control-Max-Age": "3600", } headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Content-Type": "application/json", } diff --git a/services/api/src/iss_scms_status.py b/services/api/src/iss_scms_status.py index 163e9245a..ffbbf5e25 100644 --- a/services/api/src/iss_scms_status.py +++ b/services/api/src/iss_scms_status.py @@ -2,7 +2,7 @@ import logging import common.pgquery as pgquery import common.util as util -import os +import api_environment from common.auth_tools import ( ORG_ROLE_LITERAL, @@ -49,14 +49,14 @@ def get_iss_scms_status(organization: str) -> dict: class IssScmsStatus(Resource): options_headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Access-Control-Allow-Headers": "Content-Type,Authorization,Organization", "Access-Control-Allow-Methods": "GET", "Access-Control-Max-Age": "3600", } headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Content-Type": "application/json", } diff --git a/services/api/src/main.py b/services/api/src/main.py index 383f53a1b..6b3e12030 100644 --- a/services/api/src/main.py +++ b/services/api/src/main.py @@ -1,6 +1,6 @@ from flask import Flask from flask_restful import Api -import os +import api_environment import logging # Custom script imports @@ -31,20 +31,14 @@ from contact_support import ContactSupportResource from rsu_error_summary import RSUErrorSummaryResource import smtp_error_handler +from common import common_environment -log_level = os.environ.get("LOGGING_LEVEL", "INFO") -logging.basicConfig(format="%(levelname)s:%(message)s", level=log_level) +logging.info( + "CVManager API running with LOGGING_LEVEL: " + str(common_environment.LOGGING_LEVEL) +) app = Flask(__name__) -# Feature flag environment variables -ENABLE_RSU_FEATURES = os.environ.get("ENABLE_RSU_FEATURES", "true") != "false" -ENABLE_INTERSECTION_FEATURES = ( - os.environ.get("ENABLE_INTERSECTION_FEATURES", "true") != "false" -) -ENABLE_WZDX_FEATURES = os.environ.get("ENABLE_WZDX_FEATURES", "true") != "false" -ENABLE_MOOVE_AI_FEATURES = os.environ.get("ENABLE_MOOVE_AI_FEATURES", "true") != "false" - smtp_error_handler.configure_error_emails(app) app.wsgi_app = Middleware(app.wsgi_app) @@ -53,7 +47,7 @@ @app.after_request def apply_cors_header(response): # Add CORS header to all responses to prevent webapp parsing errors. Webapps have trouble handling responses that do not have the Access-Control-Allow-Origin header set. - response.headers["Access-Control-Allow-Origin"] = os.environ["CORS_DOMAIN"] + response.headers["Access-Control-Allow-Origin"] = api_environment.CORS_DOMAIN return response @@ -69,7 +63,7 @@ def apply_cors_header(response): api.add_resource(AdminNewNotification, "/admin-new-notification") api.add_resource(ContactSupportResource, "/contact-support") -if ENABLE_RSU_FEATURES: +if api_environment.ENABLE_RSU_FEATURES: api.add_resource(RsuInfo, "/rsuinfo") api.add_resource(RsuOnlineStatus, "/rsu-online-status") api.add_resource(RsuQueryCounts, "/rsucounts") @@ -82,13 +76,13 @@ def apply_cors_header(response): api.add_resource(AdminNewRsu, "/admin-new-rsu") api.add_resource(AdminRsu, "/admin-rsu") api.add_resource(RSUErrorSummaryResource, "/rsu-error-summary") -if ENABLE_WZDX_FEATURES: +if api_environment.ENABLE_WZDX_FEATURES: api.add_resource(WzdxFeed, "/wzdx-feed") -if ENABLE_INTERSECTION_FEATURES: +if api_environment.ENABLE_INTERSECTION_FEATURES: api.add_resource(AdminNewIntersection, "/admin-new-intersection") api.add_resource(AdminIntersection, "/admin-intersection") -if ENABLE_MOOVE_AI_FEATURES: +if api_environment.ENABLE_MOOVE_AI_FEATURES: api.add_resource(MooveAiData, "/moove-ai-data") if __name__ == "__main__": - app.run(host="0.0.0.0", port=5000) + app.run(host="0.0.0.0", port=api_environment.FLASK_RUN_PORT) diff --git a/services/api/src/middleware.py b/services/api/src/middleware.py index 0ca717659..58c7d4634 100644 --- a/services/api/src/middleware.py +++ b/services/api/src/middleware.py @@ -4,7 +4,7 @@ from werkzeug.wrappers import Request from keycloak import KeycloakOpenID import logging -import os +import api_environment import jwt from werkzeug.exceptions import HTTPException, Forbidden, Unauthorized, NotImplemented @@ -25,24 +25,13 @@ class FEATURE_KEYS_LITERAL(Enum): MOOVE_AI = "moove_ai" -# Feature flag environment variables -ENABLE_RSU_FEATURES = os.getenv("ENABLE_RSU_FEATURES", "true").lower() != "false" -ENABLE_INTERSECTION_FEATURES = ( - os.getenv("ENABLE_INTERSECTION_FEATURES", "true").lower() != "false" -) -ENABLE_WZDX_FEATURES = os.getenv("ENABLE_WZDX_FEATURES", "true").lower() != "false" -ENABLE_MOOVE_AI_FEATURES = ( - os.getenv("ENABLE_MOOVE_AI_FEATURES", "true").lower() != "false" -) - - def get_user_role(token) -> UserInfo | None: # TODO: Consider using pythjon-jose or PyJWT to locally validate the token, instead of calling the Keycloak server keycloak_openid = KeycloakOpenID( - server_url=os.getenv("KEYCLOAK_ENDPOINT"), - realm_name=os.getenv("KEYCLOAK_REALM"), - client_id=os.getenv("KEYCLOAK_API_CLIENT_ID"), - client_secret_key=os.getenv("KEYCLOAK_API_CLIENT_SECRET_KEY"), + server_url=api_environment.KEYCLOAK_ENDPOINT, + realm_name=api_environment.KEYCLOAK_REALM, + client_id=api_environment.KEYCLOAK_API_CLIENT_ID, + client_secret_key=api_environment.KEYCLOAK_API_CLIENT_SECRET_KEY, ) introspect = keycloak_openid.introspect(token) data = None @@ -145,13 +134,19 @@ def is_tag_disabled(tag: FEATURE_KEYS_LITERAL | None) -> bool: Returns: bool: True if the feature should be disabled, False otherwise """ - if not ENABLE_RSU_FEATURES and tag == FEATURE_KEYS_LITERAL.RSU: + if not api_environment.ENABLE_RSU_FEATURES and tag == FEATURE_KEYS_LITERAL.RSU: return True - elif not ENABLE_INTERSECTION_FEATURES and tag == FEATURE_KEYS_LITERAL.INTERSECTION: + elif ( + not api_environment.ENABLE_INTERSECTION_FEATURES + and tag == FEATURE_KEYS_LITERAL.INTERSECTION + ): return True - elif not ENABLE_WZDX_FEATURES and tag == FEATURE_KEYS_LITERAL.WZDX: + elif not api_environment.ENABLE_WZDX_FEATURES and tag == FEATURE_KEYS_LITERAL.WZDX: return True - elif not ENABLE_MOOVE_AI_FEATURES and tag == FEATURE_KEYS_LITERAL.MOOVE_AI: + elif ( + not api_environment.ENABLE_MOOVE_AI_FEATURES + and tag == FEATURE_KEYS_LITERAL.MOOVE_AI + ): return True return False @@ -185,7 +180,7 @@ class Middleware: def __init__(self, app): self.app = app self.default_headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Content-Type": "application/json", } @@ -240,7 +235,7 @@ def __call__(self, environ, start_response): response_body, status=e.code, content_type="application/json", - headers={"Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"]}, + headers={"Access-Control-Allow-Origin": api_environment.CORS_DOMAIN}, ) return response(environ, start_response) @@ -255,6 +250,6 @@ def __call__(self, environ, start_response): response_body, status=500, content_type="application/json", - headers={"Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"]}, + headers={"Access-Control-Allow-Origin": api_environment.CORS_DOMAIN}, ) return response(environ, start_response) diff --git a/services/api/src/moove_ai_query.py b/services/api/src/moove_ai_query.py index 0b754a622..358d3a5d0 100644 --- a/services/api/src/moove_ai_query.py +++ b/services/api/src/moove_ai_query.py @@ -2,7 +2,7 @@ from flask_restful import Resource from marshmallow import Schema, fields from google.cloud import bigquery -import os +import api_environment import logging import pandas as pd from shapely import wkt @@ -18,9 +18,9 @@ def query_moove_ai(pointList): + "))" ) - client = bigquery.Client(location="US", project=os.getenv("GCP_PROJECT_ID")) - segment_agg_stats_table = os.getenv("MOOVE_AI_SEGMENT_AGG_STATS_TABLE") - segment_event_stats_table = os.getenv("MOOVE_AI_SEGMENT_EVENT_STATS_TABLE") + client = bigquery.Client(location="US", project=api_environment.GCP_PROJECT_ID) + segment_agg_stats_table = api_environment.MOOVE_AI_SEGMENT_AGG_STATS_TABLE + segment_event_stats_table = api_environment.MOOVE_AI_SEGMENT_EVENT_STATS_TABLE query = f""" SELECT sas.segment_id, @@ -77,14 +77,14 @@ class MooveAiDataSchema(Schema): class MooveAiData(Resource): options_headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Access-Control-Allow-Headers": "Content-Type,Authorization", "Access-Control-Allow-Methods": "POST", "Access-Control-Max-Age": "3600", } headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Content-Type": "application/json", } diff --git a/services/api/src/rsu_commands.py b/services/api/src/rsu_commands.py index 3767aa1b7..af1b52318 100644 --- a/services/api/src/rsu_commands.py +++ b/services/api/src/rsu_commands.py @@ -14,7 +14,7 @@ ) import ssh_commands import rsu_snmpset -import os +import api_environment # Dict of functions command_data = { @@ -176,13 +176,13 @@ class RsuCommandRequestSchema(Schema): class RsuCommandRequest(Resource): options_headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Access-Control-Allow-Headers": "Content-Type,Authorization,Organization", "Access-Control-Allow-Methods": "GET,POST", "Access-Control-Max-Age": "3600", } - headers = {"Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"]} + headers = {"Access-Control-Allow-Origin": api_environment.CORS_DOMAIN} def options(self): # CORS support diff --git a/services/api/src/rsu_error_summary.py b/services/api/src/rsu_error_summary.py index 063eae6ad..92ab19598 100644 --- a/services/api/src/rsu_error_summary.py +++ b/services/api/src/rsu_error_summary.py @@ -1,5 +1,5 @@ import logging -import os +import api_environment from flask import abort, request from flask_restful import Resource from marshmallow import Schema @@ -20,14 +20,14 @@ class RSUErrorSummarySchema(Schema): class RSUErrorSummaryResource(Resource): options_headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Access-Control-Allow-Headers": "Content-Type,Authorization", "Access-Control-Allow-Methods": "POST,OPTIONS", "Access-Control-Max-Age": "3600", } headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Access-Control-Allow-Headers": "Content-Type,Authorization", "Access-Control-Allow-Methods": "POST,OPTIONS", "Content-Type": "application/json", @@ -55,17 +55,17 @@ def post(self): for email_address in email_addresses: logging.info(f"Sending email to {email_address}") emailSender = EmailSender( - os.environ["CSM_TARGET_SMTP_SERVER_ADDRESS"], - 587, + api_environment.CSM_TARGET_SMTP_SERVER_ADDRESS, + api_environment.CSM_TARGET_SMTP_SERVER_PORT, ) emailSender.send( - sender=os.environ["CSM_EMAIL_TO_SEND_FROM"], + sender=api_environment.CSM_EMAIL_TO_SEND_FROM, recipient=email_address, subject=subject, message=message, replyEmail="", - username=os.environ["CSM_EMAIL_APP_USERNAME"], - password=os.environ["CSM_EMAIL_APP_PASSWORD"], + username=api_environment.CSM_EMAIL_APP_USERNAME, + password=api_environment.CSM_EMAIL_APP_PASSWORD, pretty=True, ) except Exception as e: diff --git a/services/api/src/rsu_geo_msg_query.py b/services/api/src/rsu_geo_msg_query.py index 3ac472497..9321ae368 100644 --- a/services/api/src/rsu_geo_msg_query.py +++ b/services/api/src/rsu_geo_msg_query.py @@ -2,12 +2,11 @@ from flask_restful import Resource from marshmallow import Schema, fields import common.util as util -import os +import api_environment import logging from datetime import datetime from pymongo import MongoClient, ASCENDING, GEOSPHERE from werkzeug.exceptions import InternalServerError - from common.auth_tools import ORG_ROLE_LITERAL, require_permission coord_resolution = 0.0001 # lats more than this are considered different @@ -32,13 +31,13 @@ def geo_hash(id, timestamp, long, lat): def get_collection(msg_type): """Get MongoDB collection based on message type""" - mongo_uri = os.getenv("MONGO_DB_URI") - db_name = os.getenv("MONGO_DB_NAME") + mongo_uri = api_environment.MONGO_DB_URI + db_name = api_environment.MONGO_DB_NAME if msg_type.lower() == "bsm": - coll_name = os.getenv("MONGO_PROCESSED_BSM_COLLECTION_NAME", "ProcessedBsm") + coll_name = api_environment.MONGO_PROCESSED_BSM_COLLECTION_NAME elif msg_type.lower() == "psm": - coll_name = os.getenv("MONGO_PROCESSED_PSM_COLLECTION_NAME", "ProcessedPsm") + coll_name = api_environment.MONGO_PROCESSED_PSM_COLLECTION_NAME else: return None, None, 400 @@ -89,10 +88,6 @@ def query_geo_data_mongo(pointList, start, end, msg_type): hashmap = {} count = 0 total_count = 0 - # If MAX_GEO_QUERY_RECORDS is not set or is an empty string, use 10000 as default - max_records = os.getenv("MAX_GEO_QUERY_RECORDS", 10000) or 10000 - # Convert to int in a separate step to avoid errors - max_records = int(max_records) try: logging.debug(f"Running filter: {filter} on mongo collection {coll_name}") @@ -117,7 +112,10 @@ def query_geo_data_mongo(pointList, start, end, msg_type): doc["geometry"]["coordinates"][1], ) - if message_hash not in hashmap and count < max_records: + if ( + message_hash not in hashmap + and count < api_environment.MAX_GEO_QUERY_RECORDS + ): doc.pop("_id") doc.pop("recordGeneratedAt") @@ -149,14 +147,14 @@ class RsuGeoDataSchema(Schema): class RsuGeoData(Resource): options_headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Access-Control-Allow-Headers": "Content-Type,Authorization", "Access-Control-Allow-Methods": "POST", "Access-Control-Max-Age": "3600", } headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Content-Type": "application/json", } diff --git a/services/api/src/rsu_geo_query.py b/services/api/src/rsu_geo_query.py index d812f648b..899ff3672 100644 --- a/services/api/src/rsu_geo_query.py +++ b/services/api/src/rsu_geo_query.py @@ -4,9 +4,8 @@ from marshmallow import Schema, fields, validate import common.pgquery as pgquery import logging -import os +import api_environment from werkzeug.exceptions import BadRequest - from common.auth_tools import ( ORG_ROLE_LITERAL, PermissionResult, @@ -97,14 +96,14 @@ class RsuGeoQuerySchema(Schema): class RsuGeoQuery(Resource): options_headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Access-Control-Allow-Headers": "Content-Type,Authorization,Organization", "Access-Control-Allow-Methods": "POST", "Access-Control-Max-Age": "3600", } headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Content-Type": "application/json", } diff --git a/services/api/src/rsu_online_status.py b/services/api/src/rsu_online_status.py index 08284fcc0..0d78c8ce9 100644 --- a/services/api/src/rsu_online_status.py +++ b/services/api/src/rsu_online_status.py @@ -8,7 +8,7 @@ import pytz import common.util as util import common.pgquery as pgquery -import os +import api_environment from common.auth_tools import ( ORG_ROLE_LITERAL, @@ -134,14 +134,14 @@ class RsuOnlineStatusSchema(Schema): class RsuOnlineStatus(Resource): options_headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Access-Control-Allow-Headers": "Content-Type,Authorization,Organization", "Access-Control-Allow-Methods": "GET", "Access-Control-Max-Age": "3600", } headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Content-Type": "application/json", } diff --git a/services/api/src/rsu_querycounts.py b/services/api/src/rsu_querycounts.py index 4913423d7..84fafbebb 100644 --- a/services/api/src/rsu_querycounts.py +++ b/services/api/src/rsu_querycounts.py @@ -5,10 +5,10 @@ from datetime import datetime, timedelta import common.pgquery as pgquery import common.util as util -import os +import api_environment import logging from pymongo import MongoClient -from werkzeug.exceptions import InternalServerError, BadRequest, Forbidden +from werkzeug.exceptions import InternalServerError, Forbidden from common.auth_tools import ( ORG_ROLE_LITERAL, @@ -18,18 +18,8 @@ generate_sql_placeholders_for_list, ) -message_types = { - "bsm": "BSM", - "map": "Map", - "spat": "SPaT", - "srm": "SRM", - "ssm": "SSM", - "tim": "TIM", - "psm": "PSM", -} - -def query_rsu_counts_mongo(allowed_ips_dict, message_type, start, end): +def query_rsu_counts_aggregated(allowed_ips_dict, start, end): start_dt = util.format_date_utc(start, "DATETIME").replace( hour=0, minute=0, second=0, microsecond=0 ) @@ -38,8 +28,10 @@ def query_rsu_counts_mongo(allowed_ips_dict, message_type, start, end): ) try: - client = MongoClient(os.getenv("MONGO_DB_URI"), serverSelectionTimeoutMS=5000) - mongo_db = client[os.getenv("MONGO_DB_NAME")] + client = MongoClient( + api_environment.MONGO_DB_URI, serverSelectionTimeoutMS=5000 + ) + mongo_db = client[api_environment.MONGO_DB_NAME] collection = mongo_db["CVCounts"] except Exception as e: logging.error( @@ -47,30 +39,59 @@ def query_rsu_counts_mongo(allowed_ips_dict, message_type, start, end): ) raise Forbidden("Failed to connect to Mongo") from e - result = {} - for rsu_ip in allowed_ips_dict: - query = { - "messageType": message_types[message_type.lower()], - "rsuIp": rsu_ip, - "timestamp": { - "$gte": start_dt, - "$lt": end_dt, - }, + # MongoDB aggregation query to get message counts by RSU IP + pipeline = [ + { + "$match": { + "rsuIp": {"$in": list(allowed_ips_dict.keys())}, + "timestamp": {"$gte": start_dt, "$lt": end_dt}, + } + }, + {"$project": {"rsuIp": 1, "messageType": 1, "count": 1, "_id": 0}}, + { + "$group": { + "_id": {"rsuIp": "$rsuIp", "messageType": "$messageType"}, + "totalCount": {"$sum": "$count"}, + } + }, + { + "$group": { + "_id": "$_id.rsuIp", + "messageTypeCounts": { + "$push": {"k": "$_id.messageType", "v": "$totalCount"} + }, + } + }, + { + "$project": { + "_id": 0, + "rsuIp": "$_id", + "messageTypeCounts": {"$arrayToObject": "$messageTypeCounts"}, + } + }, + ] + + try: + logging.debug(f"Running aggregation on {collection.name}") + cursor = collection.aggregate(pipeline, allowDiskUse=False) # Keep in memory + + # Initialize result + result = { + rsu_ip: {"road": road, "messageTypeCounts": {}} + for rsu_ip, road in allowed_ips_dict.items() } - try: - logging.debug(f"Running query: {query}, on collection: {collection.name}") - response = collection.find_one(query) - if not response: - item = {"road": allowed_ips_dict[rsu_ip], "count": 0} - else: - item = {"road": allowed_ips_dict[rsu_ip], "count": response["count"]} - result[rsu_ip] = item - except Exception as e: - logging.error(f"Filter failed: {e}") - raise InternalServerError("Encountered unknown issue") from e + # Update with actual counts + for doc in cursor: + rsu_ip = doc["rsuIp"] + if rsu_ip in result: + result[rsu_ip]["messageTypeCounts"] = doc["messageTypeCounts"] + + return result - return result + except Exception as e: + logging.error(f"Aggregation failed: {e}") + raise InternalServerError("Encountered unknown issue") from e def get_organization_rsus(user: EnvironWithOrg, qualified_orgs: list[str]): @@ -118,14 +139,14 @@ class RsuQueryCountsSchema(Schema): class RsuQueryCounts(Resource): options_headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Access-Control-Allow-Headers": "Content-Type,Authorization,Organization", "Access-Control-Allow-Methods": "GET", "Access-Control-Max-Age": "3600", } headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Content-Type": "application/json", } @@ -144,7 +165,6 @@ def get(self, permission_result: PermissionResult): if errors: abort(400, str(errors)) # Get arguments from request and set defaults if not provided - message = request.args.get("message", default="BSM") start = request.args.get( "start", default=((datetime.now() - timedelta(1)).strftime("%Y-%m-%dT%H:%M:%S")), @@ -153,18 +173,9 @@ def get(self, permission_result: PermissionResult): "end", default=((datetime.now()).strftime("%Y-%m-%dT%H:%M:%S")) ) - # Validate request with supported message types - logging.debug(f"COUNTS_MSG_TYPES: {os.getenv('COUNTS_MSG_TYPES','NOT_SET')}") - msgListStr = os.getenv("COUNTS_MSG_TYPES", "BSM,SSM,SPAT,SRM,MAP") - msgList = [msg_type.strip().title() for msg_type in msgListStr.split(",")] - if message.title() not in msgList: - raise BadRequest( - "Invalid Message Type.\nValid message types: " + ", ".join(msgList) - ) - rsu_dict = get_organization_rsus( permission_result.user, permission_result.qualified_orgs ) - data = query_rsu_counts_mongo(rsu_dict, message, start, end) + data = query_rsu_counts_aggregated(rsu_dict, start, end) return (data, 200, self.headers) diff --git a/services/api/src/rsu_querymsgfwd.py b/services/api/src/rsu_querymsgfwd.py index faca1d655..0aa26802c 100644 --- a/services/api/src/rsu_querymsgfwd.py +++ b/services/api/src/rsu_querymsgfwd.py @@ -4,7 +4,7 @@ import common.pgquery as pgquery import common.snmp.rsu_message_forward_helpers as rsu_message_forward_helpers import common.util as util -import os +import api_environment import logging from common.auth_tools import ( @@ -95,14 +95,14 @@ class RsuQueryMsgFwdSchema(Schema): class RsuQueryMsgFwd(Resource): options_headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Access-Control-Allow-Headers": "Content-Type,Authorization,Organization", "Access-Control-Allow-Methods": "GET", "Access-Control-Max-Age": "3600", } headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Content-Type": "application/json", } diff --git a/services/api/src/rsu_ssm_srm.py b/services/api/src/rsu_ssm_srm.py index 6fc4bb568..8cd774055 100644 --- a/services/api/src/rsu_ssm_srm.py +++ b/services/api/src/rsu_ssm_srm.py @@ -1,7 +1,7 @@ from typing import Any from flask_restful import Resource import common.util as util -import os +import api_environment import logging from datetime import datetime, timedelta from pymongo import MongoClient @@ -23,18 +23,11 @@ def query_ssm_data_mongo() -> list: start_utc = util.format_date_utc(start_date.isoformat()) try: - client: MongoClient = MongoClient( - os.getenv("MONGO_DB_URI"), serverSelectionTimeoutMS=5000 + client = MongoClient( + api_environment.MONGO_DB_URI, serverSelectionTimeoutMS=5000 ) - mongo_db_name = os.getenv("MONGO_DB_NAME") - srm_db_name = os.getenv("SSM_DB_NAME") - if not mongo_db_name or not srm_db_name: - logging.error( - "Missing one ore more environment variables for MongoDB: MONGO_DB_NAME, SSM_DB_NAME" - ) - raise Exception("Missing environment variables for MongoDB") - db = client[mongo_db_name] - collection = db[srm_db_name] + db = client[api_environment.MONGO_DB_NAME] + collection = db[api_environment.MONGO_SSM_COLLECTION_NAME] except Exception as e: logging.error( f"Failed to connect to Mongo counts collection with error message: {e}" @@ -88,18 +81,11 @@ def query_srm_data_mongo() -> list: start_utc = util.format_date_utc(start_date.isoformat()) try: - client: MongoClient = MongoClient( - os.getenv("MONGO_DB_URI"), serverSelectionTimeoutMS=5000 + client = MongoClient( + api_environment.MONGO_DB_URI, serverSelectionTimeoutMS=5000 ) - mongo_db_name = os.getenv("MONGO_DB_NAME") - srm_db_name = os.getenv("SRM_DB_NAME") - if not mongo_db_name or not srm_db_name: - logging.error( - "Missing one ore more environment variables for MongoDB: MONGO_DB_NAME, SSM_DB_NAME" - ) - raise Exception("Missing environment variables for MongoDB") - db = client[mongo_db_name] - collection = db[srm_db_name] + db = client[api_environment.MONGO_DB_NAME] + collection = db[api_environment.MONGO_SRM_COLLECTION_NAME] except Exception as e: logging.error( f"Failed to connect to Mongo counts collection with error message: {e}" @@ -156,14 +142,14 @@ def filter_results_by_ip_address( class RsuSsmSrmData(Resource): options_headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Access-Control-Allow-Headers": "Content-Type,Authorization", "Access-Control-Allow-Methods": "GET", "Access-Control-Max-Age": "3600", } headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Content-Type": "application/json", } diff --git a/services/api/src/rsu_upgrade.py b/services/api/src/rsu_upgrade.py index 1bcf864bc..3df428356 100644 --- a/services/api/src/rsu_upgrade.py +++ b/services/api/src/rsu_upgrade.py @@ -1,7 +1,8 @@ import common.pgquery as pgquery import json import logging -import os + +import api_environment import requests from werkzeug.exceptions import Conflict, NotImplemented @@ -44,7 +45,7 @@ def check_for_upgrade(rsu_ip): def mark_rsu_for_upgrade(rsu_ip): - if os.getenv("FIRMWARE_MANAGER_ENDPOINT") is None: + if api_environment.FIRMWARE_MANAGER_ENDPOINT is None: raise NotImplemented( # noqa: F901 "The firmware manager is not supported for this CV Manager deployment" ) @@ -52,7 +53,7 @@ def mark_rsu_for_upgrade(rsu_ip): # Verify requested target RSU is eligible for upgrade and determine next upgrade upgrade_info = check_for_upgrade(rsu_ip) - if upgrade_info["upgrade_available"] is False: + if not upgrade_info["upgrade_available"]: raise Conflict( f"Requested RSU '{rsu_ip}' is already up to date with the latest firmware" ) @@ -63,7 +64,7 @@ def mark_rsu_for_upgrade(rsu_ip): logging.info(f"Initiating firmware upgrade with the firmware manager for {rsu_ip}") # Environment variable FIRMWARE_MANAGER_ENDPOINT must contain "http://" and port - firmware_manager_endpoint = os.getenv("FIRMWARE_MANAGER_ENDPOINT") + firmware_manager_endpoint = api_environment.FIRMWARE_MANAGER_ENDPOINT post_body = {"rsu_ip": rsu_ip} response = requests.post( f"{firmware_manager_endpoint}/init_firmware_upgrade", json=post_body diff --git a/services/api/src/rsuinfo.py b/services/api/src/rsuinfo.py index 2b7c07261..64dc4ecb6 100644 --- a/services/api/src/rsuinfo.py +++ b/services/api/src/rsuinfo.py @@ -2,7 +2,7 @@ from flask_restful import Resource import logging import common.pgquery as pgquery -import os +import api_environment from common.auth_tools import ( ORG_ROLE_LITERAL, @@ -14,7 +14,6 @@ def get_rsu_data(user: EnvironWithOrg, qualified_orgs: list[str]): - # Execute the query and fetch all results query = ( "SELECT jsonb_build_object('type', 'Feature', 'id', row.rsu_id, 'geometry', ST_AsGeoJSON(row.geography)::jsonb, 'properties', to_jsonb(row)) " @@ -54,14 +53,14 @@ def get_rsu_data(user: EnvironWithOrg, qualified_orgs: list[str]): # REST endpoint resource class class RsuInfo(Resource): options_headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Access-Control-Allow-Headers": "Content-Type,Authorization,Organization", "Access-Control-Allow-Methods": "GET", "Access-Control-Max-Age": "3600", } headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Content-Type": "application/json", } diff --git a/services/api/src/smtp_error_handler.py b/services/api/src/smtp_error_handler.py index d3e1ad047..d3edf5544 100644 --- a/services/api/src/smtp_error_handler.py +++ b/services/api/src/smtp_error_handler.py @@ -7,26 +7,26 @@ import smtplib import datetime import ssl -import os +import api_environment def get_subscribed_users(): - emails = os.environ["CSM_EMAILS_TO_SEND_TO"].split(",") + emails = api_environment.CSM_EMAILS_TO_SEND_TO return emails def configure_error_emails(app): mail_handler = SMTP_SSLHandler( mailhost=[ - os.environ["CSM_TARGET_SMTP_SERVER_ADDRESS"], - int(os.environ["CSM_TARGET_SMTP_SERVER_PORT"]), + api_environment.CSM_TARGET_SMTP_SERVER_ADDRESS, + api_environment.CSM_TARGET_SMTP_SERVER_PORT, ], - fromaddr=os.environ["CSM_EMAIL_TO_SEND_FROM"], + fromaddr=api_environment.CSM_EMAIL_TO_SEND_FROM, toaddrs=[], subject="Automated CV Manager API Error", credentials=[ - os.environ["CSM_EMAIL_APP_USERNAME"], - os.environ["CSM_EMAIL_APP_PASSWORD"], + api_environment.CSM_EMAIL_APP_USERNAME, + api_environment.CSM_EMAIL_APP_PASSWORD, ], secure=(), ) @@ -64,10 +64,10 @@ def emit(self, record): body_content = open("./error_email/error_email_template.html").read() EMAIL_KEYS = { - "ENVIRONMENT": os.environ["ENVIRONMENT_NAME"], + "ENVIRONMENT": api_environment.ENVIRONMENT_NAME, "ERROR_MESSAGE": self.format(record).replace("\n", "
"), "ERROR_TIME": str(record.asctime), - "LOGS_LINK": os.environ["LOGS_LINK"], + "LOGS_LINK": api_environment.LOGS_LINK, } context = ssl._create_unverified_context() @@ -76,6 +76,8 @@ def emit(self, record): smtp.ehlo() smtp.login(self.username, self.password) + logging.debug(f"SMTP Error Email Subscribed Users: {subscribed_users}") + for email in subscribed_users: message = MIMEMultipart() message["Subject"] = self.subject diff --git a/services/api/src/userauth.py b/services/api/src/userauth.py index f9899cc46..472dbdfe6 100644 --- a/services/api/src/userauth.py +++ b/services/api/src/userauth.py @@ -1,4 +1,4 @@ -import os +import api_environment from flask_restful import Resource @@ -7,14 +7,14 @@ class UserAuth(Resource): options_headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Access-Control-Allow-Headers": "Content-Type,Authorization", "Access-Control-Allow-Methods": "GET", "Access-Control-Max-Age": "3600", } headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Content-Type": "application/json", } diff --git a/services/api/src/wzdx_feed.py b/services/api/src/wzdx_feed.py index 25e75f0b8..e5befd1e4 100644 --- a/services/api/src/wzdx_feed.py +++ b/services/api/src/wzdx_feed.py @@ -2,7 +2,9 @@ import logging import requests import json -import os +import api_environment + +from common.auth_tools import require_permission from common.auth_tools import require_permission @@ -11,7 +13,7 @@ def get_wzdx_data(): # Execute the query and fetch all results return json.loads( requests.get( - f'https://{os.getenv("WZDX_ENDPOINT")}/api/v1/wzdx?apiKey={os.getenv("WZDX_API_KEY")}' + f"https://{api_environment.WZDX_ENDPOINT}/api/v1/wzdx?apiKey={api_environment.WZDX_API_KEY}" ).content.decode("utf-8") ) @@ -19,14 +21,14 @@ def get_wzdx_data(): # REST endpoint resource class class WzdxFeed(Resource): options_headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Access-Control-Allow-Headers": "Content-Type,Authorization", "Access-Control-Allow-Methods": "GET", "Access-Control-Max-Age": "3600", } headers = { - "Access-Control-Allow-Origin": os.environ["CORS_DOMAIN"], + "Access-Control-Allow-Origin": api_environment.CORS_DOMAIN, "Content-Type": "application/json", } diff --git a/services/api/tests/src/conftest.py b/services/api/tests/src/conftest.py new file mode 100644 index 000000000..5449d4262 --- /dev/null +++ b/services/api/tests/src/conftest.py @@ -0,0 +1,13 @@ +import os + +os.environ['KEYCLOAK_ENDPOINT'] = 'keycloak-endpoint' +os.environ['KEYCLOAK_REALM'] = 'keycloak-realm' +os.environ['KEYCLOAK_API_CLIENT_ID'] = 'keycloak-api-client-id' +os.environ['KEYCLOAK_API_CLIENT_SECRET_KEY'] = 'keycloak-api-client-secret-key' +os.environ['CSM_AUTH_ENABLED'] = 'false' +os.environ['WZDX_ENDPOINT'] = 'wzdx-endpoint' +os.environ['WZDX_API_KEY'] = 'wzdx-api-key' +os.environ['GCP_PROJECT_ID'] = 'gcp-project-id' +os.environ['MOOVE_AI_SEGMENT_AGG_STATS_TABLE'] = 'moove-ai-segment-agg-stats-table' +os.environ['MOOVE_AI_SEGMENT_EVENT_STATS_TABLE'] = 'moove-ai-segment-event-stats-table' +os.environ['CORS_DOMAIN'] = 'test.com' \ No newline at end of file diff --git a/services/api/tests/src/pytest.ini b/services/api/tests/src/pytest.ini new file mode 100644 index 000000000..422492d88 --- /dev/null +++ b/services/api/tests/src/pytest.ini @@ -0,0 +1,7 @@ +# pytest.ini +[pytest] +env = + KEYCLOAK_ENDPOINT=keycloak-endpoint + KEYCLOAK_REALM=keycloak-realm + KEYCLOAK_API_CLIENT_ID=keycloak-api-client-id + KEYCLOAK_API_CLIENT_SECRET_KEY=keycloak-api-client-secret-key \ No newline at end of file diff --git a/services/api/tests/src/test_admin_intersection.py b/services/api/tests/src/test_admin_intersection.py index c1aefd0a3..5825022f3 100644 --- a/services/api/tests/src/test_admin_intersection.py +++ b/services/api/tests/src/test_admin_intersection.py @@ -6,6 +6,7 @@ from werkzeug.exceptions import HTTPException, BadRequest, InternalServerError from api.tests.data import auth_data from common.auth_tools import ENVIRON_USER_KEY +from copy import deepcopy user_valid = auth_data.get_request_environ() @@ -185,6 +186,27 @@ def test_get_intersection_data_none(mock_query_db): ) assert actual_result == expected_intersection_data +@patch("api.src.admin_intersection.pgquery.query_db") +def test_get_intersection_data_filter_by_org(mock_query_db): + mock_query_db.return_value = admin_intersection_data.get_intersection_data_return + + # Clone user_valid and assign org for this test + user_with_org = deepcopy(user_valid) + user_with_org.organization = "TestOrg" + + admin_intersection.get_intersection_data("1123", user_with_org, []) + + called_query = mock_query_db.call_args[0][0] + assert "org.name = :user_org_name" in called_query + +@patch("api.src.admin_intersection.pgquery.query_db") +def test_get_intersection_data_no_org_no_filter(mock_query_db): + mock_query_db.return_value = admin_intersection_data.get_intersection_data_return + + admin_intersection.get_intersection_data("1123", user_valid, []) + + called_query = mock_query_db.call_args[0][0] + assert "org.name = :user_org_name" not in called_query # get_modify_intersection_data @patch("api.src.admin_intersection.get_intersection_data") diff --git a/services/api/tests/src/test_admin_rsu.py b/services/api/tests/src/test_admin_rsu.py index ce178e691..65daccfe8 100644 --- a/services/api/tests/src/test_admin_rsu.py +++ b/services/api/tests/src/test_admin_rsu.py @@ -6,6 +6,7 @@ from werkzeug.exceptions import HTTPException from api.tests.data import auth_data from werkzeug.exceptions import BadRequest, InternalServerError +from copy import deepcopy user_valid = auth_data.get_request_environ() @@ -162,6 +163,27 @@ def test_get_rsu_data_none(mock_query_db): mock_query_db.assert_called_with(expected_query, params={"rsu_ip": "10.11.81.12"}) assert actual_result == expected_rsu_data +@patch("api.src.admin_rsu.pgquery.query_db") +def test_get_rsu_data_filter_by_org(mock_query_db): + mock_query_db.return_value = admin_rsu_data.get_rsu_data_return + + # Clone user_valid and assign org for this test + user_with_org = deepcopy(user_valid) + user_with_org.organization = "TestOrg" + + admin_rsu.get_rsu_data("all", user_with_org, qualified_orgs=[]) + + called_query = mock_query_db.call_args[0][0] + assert "org.name = :user_org_name" in called_query + +@patch("api.src.admin_rsu.pgquery.query_db") +def test_get_rsu_data_no_org_no_filter(mock_query_db): + mock_query_db.return_value = admin_rsu_data.get_rsu_data_return + + admin_rsu.get_rsu_data("all", user_valid, qualified_orgs=[]) + + called_query = mock_query_db.call_args[0][0] + assert "org.name = :user_org_name" not in called_query # get_modify_rsu_data @patch("api.src.admin_rsu.get_rsu_data") diff --git a/services/api/tests/src/test_admin_user.py b/services/api/tests/src/test_admin_user.py index 97e114ef7..039e8d40e 100644 --- a/services/api/tests/src/test_admin_user.py +++ b/services/api/tests/src/test_admin_user.py @@ -5,6 +5,7 @@ from sqlalchemy.exc import IntegrityError, SQLAlchemyError from api.tests.data import auth_data from werkzeug.exceptions import BadRequest, HTTPException, InternalServerError +from copy import deepcopy user_valid = auth_data.get_request_environ() @@ -160,6 +161,36 @@ def test_get_user_data_none(mock_query_db): ) assert actual_result == expected_result +@patch("api.src.admin_user.pgquery.query_db") +def test_get_user_data_filter_by_org(mock_query_db): + mock_query_db.return_value = admin_user_data.get_user_data_return + + # Clone user_valid and assign org for this test + user_with_org = deepcopy(user_valid) + user_with_org.organization = "TestOrg" + + admin_user.get_user_data( + "test@gmail.com", + user_with_org, + [], + ) + + called_query = mock_query_db.call_args[0][0] + assert "org.name = :user_org_name" in called_query + +@patch("api.src.admin_user.pgquery.query_db") +def test_get_user_data_no_org_no_filter(mock_query_db): + mock_query_db.return_value = admin_user_data.get_user_data_return + + admin_user.get_user_data( + "test@gmail.com", + user_valid, + [], + ) + + called_query = mock_query_db.call_args[0][0] + assert "org.name = :user_org_name" not in called_query + # get_modify_user_data @patch("api.src.admin_user.get_user_data") diff --git a/services/api/tests/src/test_contact_support.py b/services/api/tests/src/test_contact_support.py index 694b57a47..debab7d27 100644 --- a/services/api/tests/src/test_contact_support.py +++ b/services/api/tests/src/test_contact_support.py @@ -1,6 +1,6 @@ -import os from unittest.mock import MagicMock +from mock import patch import pytest import api.src.contact_support as contact_support @@ -19,11 +19,11 @@ def test_contact_support_schema(): exceptionOccurred = False try: schema.load(contact_support_data.contact_support_data) - except Exception as e: + except Exception: exceptionOccurred = True # assert - assert exceptionOccurred == False + assert exceptionOccurred is False def test_contact_support_schema_invalid(): @@ -39,17 +39,29 @@ def test_contact_support_schema_invalid(): # tests for ContactSupportResource class --- +@patch( + "api_environment.CSM_EMAIL_TO_SEND_FROM", + contact_support_data.CSM_EMAIL_TO_SEND_FROM, +) +@patch( + "api_environment.CSM_EMAIL_APP_USERNAME", + contact_support_data.CSM_EMAIL_APP_USERNAME, +) +@patch( + "api_environment.CSM_EMAIL_APP_PASSWORD", + contact_support_data.CSM_EMAIL_APP_PASSWORD, +) +@patch( + "api_environment.CSM_TARGET_SMTP_SERVER_ADDRESS", + DEFAULT_CSM_TARGET_SMTP_SERVER_ADDRESS, +) +@patch( + "api_environment.CSM_TARGET_SMTP_SERVER_PORT", + DEFAULT_CSM_TARGET_SMTP_SERVER_PORT, +) +@patch("api_environment.CSM_TLS_ENABLED", True) +@patch("api_environment.CSM_AUTH_ENABLED", True) def test_contact_support_resource_initialization_success(): - # prepare - os.environ["CSM_EMAIL_TO_SEND_FROM"] = contact_support_data.CSM_EMAIL_TO_SEND_FROM - os.environ["CSM_EMAIL_APP_USERNAME"] = contact_support_data.CSM_EMAIL_APP_USERNAME - os.environ["CSM_EMAIL_APP_PASSWORD"] = contact_support_data.CSM_EMAIL_APP_PASSWORD - os.environ["CSM_TARGET_SMTP_SERVER_ADDRESS"] = ( - DEFAULT_CSM_TARGET_SMTP_SERVER_ADDRESS - ) - os.environ["CSM_TARGET_SMTP_SERVER_PORT"] = str(DEFAULT_CSM_TARGET_SMTP_SERVER_PORT) - os.environ["CSM_TLS_ENABLED"] = "true" - os.environ["CSM_AUTH_ENABLED"] = "true" # execute contactSupportResource = contact_support.ContactSupportResource() @@ -64,87 +76,94 @@ def test_contact_support_resource_initialization_success(): == contact_support_data.CSM_EMAIL_APP_PASSWORD ) - # cleanup - del os.environ["CSM_EMAIL_TO_SEND_FROM"] - del os.environ["CSM_EMAIL_APP_USERNAME"] - del os.environ["CSM_EMAIL_APP_PASSWORD"] - del os.environ["CSM_TARGET_SMTP_SERVER_ADDRESS"] - del os.environ["CSM_TARGET_SMTP_SERVER_PORT"] - del os.environ["CSM_TLS_ENABLED"] - del os.environ["CSM_AUTH_ENABLED"] - +@patch("api_environment.CSM_EMAIL_TO_SEND_FROM", None) +@patch( + "api_environment.CSM_EMAIL_APP_USERNAME", + contact_support_data.CSM_EMAIL_APP_USERNAME, +) +@patch( + "api_environment.CSM_EMAIL_APP_PASSWORD", + contact_support_data.CSM_EMAIL_APP_PASSWORD, +) +@patch( + "api_environment.CSM_TARGET_SMTP_SERVER_ADDRESS", + DEFAULT_CSM_TARGET_SMTP_SERVER_ADDRESS, +) +@patch( + "api_environment.CSM_TARGET_SMTP_SERVER_PORT", + DEFAULT_CSM_TARGET_SMTP_SERVER_PORT, +) +@patch("api_environment.CSM_TLS_ENABLED", True) +@patch("api_environment.CSM_AUTH_ENABLED", True) def test_contact_support_resource_initialization_no_CSM_EMAIL_TO_SEND_FROM(): - # prepare - os.environ["CSM_EMAIL_APP_USERNAME"] = contact_support_data.CSM_EMAIL_APP_USERNAME - os.environ["CSM_EMAIL_APP_PASSWORD"] = contact_support_data.CSM_EMAIL_APP_PASSWORD - os.environ["CSM_TARGET_SMTP_SERVER_ADDRESS"] = ( - DEFAULT_CSM_TARGET_SMTP_SERVER_ADDRESS - ) - os.environ["CSM_TARGET_SMTP_SERVER_PORT"] = str(DEFAULT_CSM_TARGET_SMTP_SERVER_PORT) - os.environ["CSM_TLS_ENABLED"] = "true" - os.environ["CSM_AUTH_ENABLED"] = "true" - # execute exceptionOccurred = False try: - contactSupportResource = contact_support.ContactSupportResource() - except Exception as e: + contact_support.ContactSupportResource() + except Exception: exceptionOccurred = True # assert assert exceptionOccurred # cleanup - del os.environ["CSM_EMAIL_APP_USERNAME"] - del os.environ["CSM_EMAIL_APP_PASSWORD"] - del os.environ["CSM_TARGET_SMTP_SERVER_ADDRESS"] - del os.environ["CSM_TARGET_SMTP_SERVER_PORT"] - del os.environ["CSM_TLS_ENABLED"] - del os.environ["CSM_AUTH_ENABLED"] +@patch( + "api_environment.CSM_EMAIL_TO_SEND_FROM", + contact_support_data.CSM_EMAIL_TO_SEND_FROM, +) +@patch( + "api_environment.CSM_EMAIL_APP_USERNAME", + contact_support_data.CSM_EMAIL_APP_USERNAME, +) +@patch("api_environment.CSM_EMAIL_APP_PASSWORD", None) +@patch( + "api_environment.CSM_TARGET_SMTP_SERVER_ADDRESS", + DEFAULT_CSM_TARGET_SMTP_SERVER_ADDRESS, +) +@patch( + "api_environment.CSM_TARGET_SMTP_SERVER_PORT", + DEFAULT_CSM_TARGET_SMTP_SERVER_PORT, +) +@patch("api_environment.CSM_TLS_ENABLED", True) +@patch("api_environment.CSM_AUTH_ENABLED", True) def test_contact_support_resource_initialization_no_CSM_EMAIL_APP_PASSWORD(): - # prepare - os.environ["CSM_EMAIL_TO_SEND_FROM"] = contact_support_data.CSM_EMAIL_TO_SEND_FROM - os.environ["CSM_EMAIL_APP_USERNAME"] = contact_support_data.CSM_EMAIL_APP_USERNAME - os.environ["CSM_TARGET_SMTP_SERVER_ADDRESS"] = ( - DEFAULT_CSM_TARGET_SMTP_SERVER_ADDRESS - ) - os.environ["CSM_TARGET_SMTP_SERVER_PORT"] = str(DEFAULT_CSM_TARGET_SMTP_SERVER_PORT) - os.environ["CSM_TLS_ENABLED"] = "true" - os.environ["CSM_AUTH_ENABLED"] = "true" - # execute exceptionOccurred = False try: - contactSupportResource = contact_support.ContactSupportResource() - except Exception as e: + contact_support.ContactSupportResource() + except Exception: exceptionOccurred = True # assert assert exceptionOccurred - # cleanup - del os.environ["CSM_EMAIL_TO_SEND_FROM"] - del os.environ["CSM_EMAIL_APP_USERNAME"] - del os.environ["CSM_TARGET_SMTP_SERVER_ADDRESS"] - del os.environ["CSM_TARGET_SMTP_SERVER_PORT"] - del os.environ["CSM_TLS_ENABLED"] - del os.environ["CSM_AUTH_ENABLED"] - +@patch( + "api_environment.CSM_EMAIL_TO_SEND_FROM", + contact_support_data.CSM_EMAIL_TO_SEND_FROM, +) +@patch( + "api_environment.CSM_EMAIL_APP_USERNAME", + contact_support_data.CSM_EMAIL_APP_USERNAME, +) +@patch( + "api_environment.CSM_EMAIL_APP_PASSWORD", + contact_support_data.CSM_EMAIL_APP_PASSWORD, +) +@patch( + "api_environment.CSM_TARGET_SMTP_SERVER_ADDRESS", + DEFAULT_CSM_TARGET_SMTP_SERVER_ADDRESS, +) +@patch( + "api_environment.CSM_TARGET_SMTP_SERVER_PORT", + DEFAULT_CSM_TARGET_SMTP_SERVER_PORT, +) +@patch("api_environment.CSM_TLS_ENABLED", True) +@patch("api_environment.CSM_AUTH_ENABLED", True) def test_options(): - # prepare - os.environ["CSM_EMAIL_TO_SEND_FROM"] = contact_support_data.CSM_EMAIL_TO_SEND_FROM - os.environ["CSM_TARGET_SMTP_SERVER_ADDRESS"] = ( - DEFAULT_CSM_TARGET_SMTP_SERVER_ADDRESS - ) - os.environ["CSM_TARGET_SMTP_SERVER_PORT"] = str(DEFAULT_CSM_TARGET_SMTP_SERVER_PORT) - os.environ["CSM_TLS_ENABLED"] = "true" - os.environ["CSM_AUTH_ENABLED"] = "true" - os.environ["CSM_EMAIL_APP_USERNAME"] = contact_support_data.CSM_EMAIL_APP_USERNAME - os.environ["CSM_EMAIL_APP_PASSWORD"] = contact_support_data.CSM_EMAIL_APP_PASSWORD contactSupportResource = contact_support.ContactSupportResource() # execute @@ -153,27 +172,30 @@ def test_options(): # assert assert result == ("", 204, contactSupportResource.options_headers) - # cleanup - del os.environ["CSM_EMAIL_TO_SEND_FROM"] - del os.environ["CSM_EMAIL_APP_USERNAME"] - del os.environ["CSM_EMAIL_APP_PASSWORD"] - del os.environ["CSM_TARGET_SMTP_SERVER_ADDRESS"] - del os.environ["CSM_TARGET_SMTP_SERVER_PORT"] - del os.environ["CSM_TLS_ENABLED"] - del os.environ["CSM_AUTH_ENABLED"] - +@patch( + "api_environment.CSM_EMAIL_TO_SEND_FROM", + contact_support_data.CSM_EMAIL_TO_SEND_FROM, +) +@patch( + "api_environment.CSM_EMAIL_APP_USERNAME", + contact_support_data.CSM_EMAIL_APP_USERNAME, +) +@patch( + "api_environment.CSM_EMAIL_APP_PASSWORD", + contact_support_data.CSM_EMAIL_APP_PASSWORD, +) +@patch( + "api_environment.CSM_TARGET_SMTP_SERVER_ADDRESS", + DEFAULT_CSM_TARGET_SMTP_SERVER_ADDRESS, +) +@patch( + "api_environment.CSM_TARGET_SMTP_SERVER_PORT", + DEFAULT_CSM_TARGET_SMTP_SERVER_PORT, +) +@patch("api_environment.CSM_TLS_ENABLED", True) +@patch("api_environment.CSM_AUTH_ENABLED", True) def test_post_success(): - # prepare - os.environ["CSM_EMAIL_TO_SEND_FROM"] = contact_support_data.CSM_EMAIL_TO_SEND_FROM - os.environ["CSM_EMAIL_APP_USERNAME"] = contact_support_data.CSM_EMAIL_APP_USERNAME - os.environ["CSM_EMAIL_APP_PASSWORD"] = contact_support_data.CSM_EMAIL_APP_PASSWORD - os.environ["CSM_TARGET_SMTP_SERVER_ADDRESS"] = ( - DEFAULT_CSM_TARGET_SMTP_SERVER_ADDRESS - ) - os.environ["CSM_TARGET_SMTP_SERVER_PORT"] = str(DEFAULT_CSM_TARGET_SMTP_SERVER_PORT) - os.environ["CSM_TLS_ENABLED"] = "true" - os.environ["CSM_AUTH_ENABLED"] = "true" contactSupportResource = contact_support.ContactSupportResource() contactSupportResource.validate_input = MagicMock() contactSupportResource.send = MagicMock() @@ -186,27 +208,30 @@ def test_post_success(): # assert assert result == ("", 200, contactSupportResource.headers) - # cleanup - del os.environ["CSM_EMAIL_TO_SEND_FROM"] - del os.environ["CSM_EMAIL_APP_USERNAME"] - del os.environ["CSM_EMAIL_APP_PASSWORD"] - del os.environ["CSM_TARGET_SMTP_SERVER_ADDRESS"] - del os.environ["CSM_TARGET_SMTP_SERVER_PORT"] - del os.environ["CSM_TLS_ENABLED"] - del os.environ["CSM_AUTH_ENABLED"] - +@patch( + "api_environment.CSM_EMAIL_TO_SEND_FROM", + contact_support_data.CSM_EMAIL_TO_SEND_FROM, +) +@patch( + "api_environment.CSM_EMAIL_APP_USERNAME", + contact_support_data.CSM_EMAIL_APP_USERNAME, +) +@patch( + "api_environment.CSM_EMAIL_APP_PASSWORD", + contact_support_data.CSM_EMAIL_APP_PASSWORD, +) +@patch( + "api_environment.CSM_TARGET_SMTP_SERVER_ADDRESS", + DEFAULT_CSM_TARGET_SMTP_SERVER_ADDRESS, +) +@patch( + "api_environment.CSM_TARGET_SMTP_SERVER_PORT", + DEFAULT_CSM_TARGET_SMTP_SERVER_PORT, +) +@patch("api_environment.CSM_TLS_ENABLED", True) +@patch("api_environment.CSM_AUTH_ENABLED", True) def test_post_no_json_body(): - # prepare - os.environ["CSM_EMAIL_TO_SEND_FROM"] = contact_support_data.CSM_EMAIL_TO_SEND_FROM - os.environ["CSM_EMAIL_APP_USERNAME"] = contact_support_data.CSM_EMAIL_APP_USERNAME - os.environ["CSM_EMAIL_APP_PASSWORD"] = contact_support_data.CSM_EMAIL_APP_PASSWORD - os.environ["CSM_TARGET_SMTP_SERVER_ADDRESS"] = ( - DEFAULT_CSM_TARGET_SMTP_SERVER_ADDRESS - ) - os.environ["CSM_TARGET_SMTP_SERVER_PORT"] = str(DEFAULT_CSM_TARGET_SMTP_SERVER_PORT) - os.environ["CSM_TLS_ENABLED"] = "true" - os.environ["CSM_AUTH_ENABLED"] = "true" contactSupportResource = contact_support.ContactSupportResource() contactSupportResource.validate_input = MagicMock() contactSupportResource.send = MagicMock() @@ -221,27 +246,30 @@ def test_post_no_json_body(): assert contact_support.abort.call_count == 2 assert result == ("", 200, contactSupportResource.headers) - # cleanup - del os.environ["CSM_EMAIL_TO_SEND_FROM"] - del os.environ["CSM_EMAIL_APP_USERNAME"] - del os.environ["CSM_EMAIL_APP_PASSWORD"] - del os.environ["CSM_TARGET_SMTP_SERVER_ADDRESS"] - del os.environ["CSM_TARGET_SMTP_SERVER_PORT"] - del os.environ["CSM_TLS_ENABLED"] - del os.environ["CSM_AUTH_ENABLED"] - +@patch( + "api_environment.CSM_EMAIL_TO_SEND_FROM", + contact_support_data.CSM_EMAIL_TO_SEND_FROM, +) +@patch( + "api_environment.CSM_EMAIL_APP_USERNAME", + contact_support_data.CSM_EMAIL_APP_USERNAME, +) +@patch( + "api_environment.CSM_EMAIL_APP_PASSWORD", + contact_support_data.CSM_EMAIL_APP_PASSWORD, +) +@patch( + "api_environment.CSM_TARGET_SMTP_SERVER_ADDRESS", + DEFAULT_CSM_TARGET_SMTP_SERVER_ADDRESS, +) +@patch( + "api_environment.CSM_TARGET_SMTP_SERVER_PORT", + DEFAULT_CSM_TARGET_SMTP_SERVER_PORT, +) +@patch("api_environment.CSM_TLS_ENABLED", True) +@patch("api_environment.CSM_AUTH_ENABLED", True) def test_validate_input(): - # prepare - os.environ["CSM_EMAIL_TO_SEND_FROM"] = contact_support_data.CSM_EMAIL_TO_SEND_FROM - os.environ["CSM_EMAIL_APP_USERNAME"] = contact_support_data.CSM_EMAIL_APP_USERNAME - os.environ["CSM_EMAIL_APP_PASSWORD"] = contact_support_data.CSM_EMAIL_APP_PASSWORD - os.environ["CSM_TARGET_SMTP_SERVER_ADDRESS"] = ( - DEFAULT_CSM_TARGET_SMTP_SERVER_ADDRESS - ) - os.environ["CSM_TARGET_SMTP_SERVER_PORT"] = str(DEFAULT_CSM_TARGET_SMTP_SERVER_PORT) - os.environ["CSM_TLS_ENABLED"] = "true" - os.environ["CSM_AUTH_ENABLED"] = "true" contactSupportResource = contact_support.ContactSupportResource() # execute @@ -250,16 +278,4 @@ def test_validate_input(): ) # assert - assert result == None - - # cleanup - del os.environ["CSM_EMAIL_TO_SEND_FROM"] - del os.environ["CSM_EMAIL_APP_USERNAME"] - del os.environ["CSM_EMAIL_APP_PASSWORD"] - del os.environ["CSM_TARGET_SMTP_SERVER_ADDRESS"] - del os.environ["CSM_TARGET_SMTP_SERVER_PORT"] - del os.environ["CSM_TLS_ENABLED"] - del os.environ["CSM_AUTH_ENABLED"] - - -# end of tests for ContactSupportResource class --- + assert result is None diff --git a/services/api/tests/src/test_middleware.py b/services/api/tests/src/test_middleware.py index 53d518c32..0ed3cb8ef 100644 --- a/services/api/tests/src/test_middleware.py +++ b/services/api/tests/src/test_middleware.py @@ -1,5 +1,4 @@ from unittest.mock import patch, Mock - from api.src import middleware from api.tests.data import auth_data from werkzeug.exceptions import Unauthorized @@ -159,40 +158,48 @@ def test_middleware_class_call_contact_support(mock_request, mock_get_user_role) mock_request.assert_called_once_with(environ) -@patch("api.src.middleware.ENABLE_RSU_FEATURES", True) -@patch("api.src.middleware.ENABLE_INTERSECTION_FEATURES", True) -@patch("api.src.middleware.ENABLE_WZDX_FEATURES", True) +@patch("api_environment.ENABLE_RSU_FEATURES", True) +@patch("api_environment.ENABLE_INTERSECTION_FEATURES", True) +@patch("api_environment.ENABLE_WZDX_FEATURES", True) def test_evaluate_tag_all_enabled(): - assert not middleware.is_tag_disabled(middleware.FEATURE_KEYS_LITERAL.RSU) - assert not middleware.is_tag_disabled(middleware.FEATURE_KEYS_LITERAL.INTERSECTION) - assert not middleware.is_tag_disabled(middleware.FEATURE_KEYS_LITERAL.WZDX) + assert middleware.is_tag_disabled(middleware.FEATURE_KEYS_LITERAL.RSU) is False + assert ( + middleware.is_tag_disabled(middleware.FEATURE_KEYS_LITERAL.INTERSECTION) + is False + ) + assert middleware.is_tag_disabled(middleware.FEATURE_KEYS_LITERAL.WZDX) is False -@patch("api.src.middleware.ENABLE_RSU_FEATURES", False) -@patch("api.src.middleware.ENABLE_INTERSECTION_FEATURES", False) -@patch("api.src.middleware.ENABLE_WZDX_FEATURES", False) +@patch("api_environment.ENABLE_RSU_FEATURES", False) +@patch("api_environment.ENABLE_INTERSECTION_FEATURES", False) +@patch("api_environment.ENABLE_WZDX_FEATURES", False) def test_evaluate_tag_all_disabled(): from api.src import middleware as middleware - assert middleware.is_tag_disabled(middleware.FEATURE_KEYS_LITERAL.RSU) - assert middleware.is_tag_disabled(middleware.FEATURE_KEYS_LITERAL.INTERSECTION) - assert middleware.is_tag_disabled(middleware.FEATURE_KEYS_LITERAL.WZDX) + assert middleware.is_tag_disabled(middleware.FEATURE_KEYS_LITERAL.RSU) is True + assert ( + middleware.is_tag_disabled(middleware.FEATURE_KEYS_LITERAL.INTERSECTION) is True + ) + assert middleware.is_tag_disabled(middleware.FEATURE_KEYS_LITERAL.WZDX) is True -@patch("api.src.middleware.ENABLE_RSU_FEATURES", False) -@patch("api.src.middleware.ENABLE_INTERSECTION_FEATURES", True) -@patch("api.src.middleware.ENABLE_WZDX_FEATURES", False) +@patch("api_environment.ENABLE_RSU_FEATURES", False) +@patch("api_environment.ENABLE_INTERSECTION_FEATURES", True) +@patch("api_environment.ENABLE_WZDX_FEATURES", False) def test_evaluate_tag_different(): from api.src import middleware as middleware - assert middleware.is_tag_disabled(middleware.FEATURE_KEYS_LITERAL.RSU) - assert not middleware.is_tag_disabled(middleware.FEATURE_KEYS_LITERAL.INTERSECTION) - assert middleware.is_tag_disabled(middleware.FEATURE_KEYS_LITERAL.WZDX) + assert middleware.is_tag_disabled(middleware.FEATURE_KEYS_LITERAL.RSU) is True + assert ( + middleware.is_tag_disabled(middleware.FEATURE_KEYS_LITERAL.INTERSECTION) + is False + ) + assert middleware.is_tag_disabled(middleware.FEATURE_KEYS_LITERAL.WZDX) is True -@patch("api.src.middleware.ENABLE_RSU_FEATURES", False) -@patch("api.src.middleware.ENABLE_INTERSECTION_FEATURES", False) -@patch("api.src.middleware.ENABLE_WZDX_FEATURES", False) +@patch("api_environment.ENABLE_RSU_FEATURES", False) +@patch("api_environment.ENABLE_INTERSECTION_FEATURES", False) +@patch("api_environment.ENABLE_WZDX_FEATURES", False) def test_is_feature_disabled_disabled(): from api.src import middleware as middleware @@ -203,16 +210,16 @@ def test_is_feature_disabled_disabled(): "/d": None, } - assert middleware.is_endpoint_disabled(feature_tags, "/a") - assert middleware.is_endpoint_disabled(feature_tags, "/b") - assert middleware.is_endpoint_disabled(feature_tags, "/c") - assert not middleware.is_endpoint_disabled(feature_tags, "/d") - assert not middleware.is_endpoint_disabled(feature_tags, "/f") + assert middleware.is_endpoint_disabled(feature_tags, "/a") is True + assert middleware.is_endpoint_disabled(feature_tags, "/b") is True + assert middleware.is_endpoint_disabled(feature_tags, "/c") is True + assert middleware.is_endpoint_disabled(feature_tags, "/d") is False + assert middleware.is_endpoint_disabled(feature_tags, "/f") is False -@patch("api.src.middleware.ENABLE_RSU_FEATURES", True) -@patch("api.src.middleware.ENABLE_INTERSECTION_FEATURES", True) -@patch("api.src.middleware.ENABLE_WZDX_FEATURES", True) +@patch("api_environment.ENABLE_RSU_FEATURES", True) +@patch("api_environment.ENABLE_INTERSECTION_FEATURES", True) +@patch("api_environment.ENABLE_WZDX_FEATURES", True) def test_is_feature_disabled_enabled(): from api.src import middleware as middleware @@ -223,16 +230,16 @@ def test_is_feature_disabled_enabled(): "/d": None, } - assert not middleware.is_endpoint_disabled(feature_tags, "/a") - assert not middleware.is_endpoint_disabled(feature_tags, "/b") - assert not middleware.is_endpoint_disabled(feature_tags, "/c") - assert not middleware.is_endpoint_disabled(feature_tags, "/d") - assert not middleware.is_endpoint_disabled(feature_tags, "/f") + assert middleware.is_endpoint_disabled(feature_tags, "/a") is False + assert middleware.is_endpoint_disabled(feature_tags, "/b") is False + assert middleware.is_endpoint_disabled(feature_tags, "/c") is False + assert middleware.is_endpoint_disabled(feature_tags, "/d") is False + assert middleware.is_endpoint_disabled(feature_tags, "/f") is False -@patch("api.src.middleware.ENABLE_RSU_FEATURES", True) -@patch("api.src.middleware.ENABLE_INTERSECTION_FEATURES", False) -@patch("api.src.middleware.ENABLE_WZDX_FEATURES", False) +@patch("api_environment.ENABLE_RSU_FEATURES", True) +@patch("api_environment.ENABLE_INTERSECTION_FEATURES", False) +@patch("api_environment.ENABLE_WZDX_FEATURES", False) def test_is_feature_disabled_different(): from api.src import middleware as middleware @@ -243,8 +250,8 @@ def test_is_feature_disabled_different(): "/d": None, } - assert not middleware.is_endpoint_disabled(feature_tags, "/a") - assert middleware.is_endpoint_disabled(feature_tags, "/b") - assert middleware.is_endpoint_disabled(feature_tags, "/c") - assert not middleware.is_endpoint_disabled(feature_tags, "/d") - assert not middleware.is_endpoint_disabled(feature_tags, "/f") + assert middleware.is_endpoint_disabled(feature_tags, "/a") is False + assert middleware.is_endpoint_disabled(feature_tags, "/b") is True + assert middleware.is_endpoint_disabled(feature_tags, "/c") is True + assert middleware.is_endpoint_disabled(feature_tags, "/d") is False + assert middleware.is_endpoint_disabled(feature_tags, "/f") is False diff --git a/services/api/tests/src/test_moove_ai_query.py b/services/api/tests/src/test_moove_ai_query.py index 27ad8e5ca..60293c751 100644 --- a/services/api/tests/src/test_moove_ai_query.py +++ b/services/api/tests/src/test_moove_ai_query.py @@ -3,7 +3,6 @@ import pytest import api.src.moove_ai_query as moove_ai_query import api.tests.data.moove_ai_query_data as moove_ai_query_data -import os import pandas as pd from werkzeug.exceptions import InternalServerError, BadRequest @@ -51,13 +50,14 @@ def test_entry_post_bad_req(mock_query_moove_ai): ###################################### Testing Functions ########################################## -@patch.dict( - os.environ, - { - "GCP_PROJECT_ID": "test_project", - "MOOVE_AI_SEGMENT_AGG_STATS_TABLE": "test_segment_agg_stats_table", - "MOOVE_AI_SEGMENT_EVENT_STATS_TABLE": "test_segment_event_stats_table", - }, +@patch("api_environment.GCP_PROJECT_ID", "test_project") +@patch( + "api_environment.MOOVE_AI_SEGMENT_AGG_STATS_TABLE", + "test_segment_agg_stats_table", +) +@patch( + "api_environment.MOOVE_AI_SEGMENT_EVENT_STATS_TABLE", + "test_segment_event_stats_table", ) @patch("api.src.moove_ai_query.bigquery") def test_query_moove_ai(mock_bigquery): @@ -75,13 +75,14 @@ def test_query_moove_ai(mock_bigquery): assert resp == moove_ai_query_data.feature_list -@patch.dict( - os.environ, - { - "GCP_PROJECT_ID": "test_project", - "MOOVE_AI_SEGMENT_AGG_STATS_TABLE": "test_segment_agg_stats_table", - "MOOVE_AI_SEGMENT_EVENT_STATS_TABLE": "test_segment_event_stats_table", - }, +@patch("api_environment.GCP_PROJECT_ID", "test_project") +@patch( + "api_environment.MOOVE_AI_SEGMENT_AGG_STATS_TABLE", + "test_segment_agg_stats_table", +) +@patch( + "api_environment.MOOVE_AI_SEGMENT_EVENT_STATS_TABLE", + "test_segment_event_stats_table", ) @patch("api.src.moove_ai_query.bigquery") def test_query_moove_ai_exception(mock_bigquery): diff --git a/services/api/tests/src/test_rsu_error_summary.py b/services/api/tests/src/test_rsu_error_summary.py index a93b35d9a..d1438b659 100644 --- a/services/api/tests/src/test_rsu_error_summary.py +++ b/services/api/tests/src/test_rsu_error_summary.py @@ -1,6 +1,7 @@ -import os from unittest.mock import MagicMock +from mock import patch + import api.src.rsu_error_summary as rsu_error_summary import api.tests.data.rsu_error_summary_data as rsu_error_summary_data @@ -14,11 +15,11 @@ def test_rsu_error_summary_schema(): exceptionOccurred = False try: schema.load(rsu_error_summary_data.rsu_error_summary_data_good) - except Exception as e: + except Exception: exceptionOccurred = True # assert - assert exceptionOccurred == False + assert exceptionOccurred is False def test_rsu_error_summary_schema_invalid(): @@ -29,25 +30,37 @@ def test_rsu_error_summary_schema_invalid(): exceptionOccurred = False try: schema.load(rsu_error_summary_data.rsu_error_summary_data_bad) - except Exception as e: + except Exception: exceptionOccurred = True # assert - assert exceptionOccurred == True + assert exceptionOccurred is True # RSUErrorSummaryResource class tests --- +@patch( + "api_environment.CSM_EMAIL_TO_SEND_FROM", + rsu_error_summary_data.CSM_EMAIL_TO_SEND_FROM, +) +@patch( + "api_environment.CSM_EMAIL_APP_USERNAME", + rsu_error_summary_data.CSM_EMAIL_APP_USERNAME, +) +@patch( + "api_environment.CSM_EMAIL_APP_PASSWORD", + rsu_error_summary_data.CSM_EMAIL_APP_PASSWORD, +) +@patch( + "api_environment.CSM_TARGET_SMTP_SERVER_ADDRESS", + rsu_error_summary_data.DEFAULT_CSM_TARGET_SMTP_SERVER_ADDRESS, +) +@patch( + "api_environment.CSM_TARGET_SMTP_SERVER_PORT", + rsu_error_summary_data.DEFAULT_CSM_TARGET_SMTP_SERVER_PORT, +) +@patch("api_environment.CSM_TLS_ENABLED", True) +@patch("api_environment.CSM_AUTH_ENABLED", True) def test_options(): - # prepare - os.environ["CSM_EMAIL_TO_SEND_FROM"] = rsu_error_summary_data.CSM_EMAIL_TO_SEND_FROM - os.environ["CSM_EMAIL_APP_USERNAME"] = rsu_error_summary_data.CSM_EMAIL_APP_USERNAME - os.environ["CSM_EMAIL_APP_PASSWORD"] = rsu_error_summary_data.CSM_EMAIL_APP_PASSWORD - os.environ["CSM_TARGET_SMTP_SERVER_ADDRESS"] = ( - rsu_error_summary_data.DEFAULT_CSM_TARGET_SMTP_SERVER_ADDRESS - ) - os.environ["CSM_TARGET_SMTP_SERVER_PORT"] = str( - rsu_error_summary_data.DEFAULT_CSM_TARGET_SMTP_SERVER_PORT - ) RSUErrorSummaryResource = rsu_error_summary.RSUErrorSummaryResource() # execute @@ -56,25 +69,30 @@ def test_options(): # assert assert result == ("", 204, RSUErrorSummaryResource.options_headers) - # cleanup - del os.environ["CSM_EMAIL_TO_SEND_FROM"] - del os.environ["CSM_EMAIL_APP_USERNAME"] - del os.environ["CSM_EMAIL_APP_PASSWORD"] - del os.environ["CSM_TARGET_SMTP_SERVER_ADDRESS"] - del os.environ["CSM_TARGET_SMTP_SERVER_PORT"] - +@patch( + "api_environment.CSM_EMAIL_TO_SEND_FROM", + rsu_error_summary_data.CSM_EMAIL_TO_SEND_FROM, +) +@patch( + "api_environment.CSM_EMAIL_APP_USERNAME", + rsu_error_summary_data.CSM_EMAIL_APP_USERNAME, +) +@patch( + "api_environment.CSM_EMAIL_APP_PASSWORD", + rsu_error_summary_data.CSM_EMAIL_APP_PASSWORD, +) +@patch( + "api_environment.CSM_TARGET_SMTP_SERVER_ADDRESS", + rsu_error_summary_data.DEFAULT_CSM_TARGET_SMTP_SERVER_ADDRESS, +) +@patch( + "api_environment.CSM_TARGET_SMTP_SERVER_PORT", + rsu_error_summary_data.DEFAULT_CSM_TARGET_SMTP_SERVER_PORT, +) +@patch("api_environment.CSM_TLS_ENABLED", True) +@patch("api_environment.CSM_AUTH_ENABLED", True) def test_post_success(): - # prepare - os.environ["CSM_EMAIL_TO_SEND_FROM"] = rsu_error_summary_data.CSM_EMAIL_TO_SEND_FROM - os.environ["CSM_EMAIL_APP_USERNAME"] = rsu_error_summary_data.CSM_EMAIL_APP_USERNAME - os.environ["CSM_EMAIL_APP_PASSWORD"] = rsu_error_summary_data.CSM_EMAIL_APP_PASSWORD - os.environ["CSM_TARGET_SMTP_SERVER_ADDRESS"] = ( - rsu_error_summary_data.DEFAULT_CSM_TARGET_SMTP_SERVER_ADDRESS - ) - os.environ["CSM_TARGET_SMTP_SERVER_PORT"] = str( - rsu_error_summary_data.DEFAULT_CSM_TARGET_SMTP_SERVER_PORT - ) RSUErrorSummaryResource = rsu_error_summary.RSUErrorSummaryResource() RSUErrorSummaryResource.validate_input = MagicMock() RSUErrorSummaryResource.send = MagicMock() @@ -87,25 +105,30 @@ def test_post_success(): # assert assert result == ("", 200, RSUErrorSummaryResource.headers) - # cleanup - del os.environ["CSM_EMAIL_TO_SEND_FROM"] - del os.environ["CSM_EMAIL_APP_USERNAME"] - del os.environ["CSM_EMAIL_APP_PASSWORD"] - del os.environ["CSM_TARGET_SMTP_SERVER_ADDRESS"] - del os.environ["CSM_TARGET_SMTP_SERVER_PORT"] - +@patch( + "api_environment.CSM_EMAIL_TO_SEND_FROM", + rsu_error_summary_data.CSM_EMAIL_TO_SEND_FROM, +) +@patch( + "api_environment.CSM_EMAIL_APP_USERNAME", + rsu_error_summary_data.CSM_EMAIL_APP_USERNAME, +) +@patch( + "api_environment.CSM_EMAIL_APP_PASSWORD", + rsu_error_summary_data.CSM_EMAIL_APP_PASSWORD, +) +@patch( + "api_environment.CSM_TARGET_SMTP_SERVER_ADDRESS", + rsu_error_summary_data.DEFAULT_CSM_TARGET_SMTP_SERVER_ADDRESS, +) +@patch( + "api_environment.CSM_TARGET_SMTP_SERVER_PORT", + rsu_error_summary_data.DEFAULT_CSM_TARGET_SMTP_SERVER_PORT, +) +@patch("api_environment.CSM_TLS_ENABLED", True) +@patch("api_environment.CSM_AUTH_ENABLED", True) def test_post_no_json_body(): - # prepare - os.environ["CSM_EMAIL_TO_SEND_FROM"] = rsu_error_summary_data.CSM_EMAIL_TO_SEND_FROM - os.environ["CSM_EMAIL_APP_USERNAME"] = rsu_error_summary_data.CSM_EMAIL_APP_USERNAME - os.environ["CSM_EMAIL_APP_PASSWORD"] = rsu_error_summary_data.CSM_EMAIL_APP_PASSWORD - os.environ["CSM_TARGET_SMTP_SERVER_ADDRESS"] = ( - rsu_error_summary_data.DEFAULT_CSM_TARGET_SMTP_SERVER_ADDRESS - ) - os.environ["CSM_TARGET_SMTP_SERVER_PORT"] = str( - rsu_error_summary_data.DEFAULT_CSM_TARGET_SMTP_SERVER_PORT - ) RSUErrorSummaryResource = rsu_error_summary.RSUErrorSummaryResource() RSUErrorSummaryResource.validate_input = MagicMock() RSUErrorSummaryResource.send = MagicMock() @@ -120,25 +143,30 @@ def test_post_no_json_body(): assert rsu_error_summary.abort.call_count == 2 assert result == ("", 200, RSUErrorSummaryResource.headers) - # cleanup - del os.environ["CSM_EMAIL_TO_SEND_FROM"] - del os.environ["CSM_EMAIL_APP_USERNAME"] - del os.environ["CSM_EMAIL_APP_PASSWORD"] - del os.environ["CSM_TARGET_SMTP_SERVER_ADDRESS"] - del os.environ["CSM_TARGET_SMTP_SERVER_PORT"] - +@patch( + "api_environment.CSM_EMAIL_TO_SEND_FROM", + rsu_error_summary_data.CSM_EMAIL_TO_SEND_FROM, +) +@patch( + "api_environment.CSM_EMAIL_APP_USERNAME", + rsu_error_summary_data.CSM_EMAIL_APP_USERNAME, +) +@patch( + "api_environment.CSM_EMAIL_APP_PASSWORD", + rsu_error_summary_data.CSM_EMAIL_APP_PASSWORD, +) +@patch( + "api_environment.CSM_TARGET_SMTP_SERVER_ADDRESS", + rsu_error_summary_data.DEFAULT_CSM_TARGET_SMTP_SERVER_ADDRESS, +) +@patch( + "api_environment.CSM_TARGET_SMTP_SERVER_PORT", + rsu_error_summary_data.DEFAULT_CSM_TARGET_SMTP_SERVER_PORT, +) +@patch("api_environment.CSM_TLS_ENABLED", True) +@patch("api_environment.CSM_AUTH_ENABLED", True) def test_validate_input_good(): - # prepare - os.environ["CSM_EMAIL_TO_SEND_FROM"] = rsu_error_summary_data.CSM_EMAIL_TO_SEND_FROM - os.environ["CSM_EMAIL_APP_USERNAME"] = rsu_error_summary_data.CSM_EMAIL_APP_USERNAME - os.environ["CSM_EMAIL_APP_PASSWORD"] = rsu_error_summary_data.CSM_EMAIL_APP_PASSWORD - os.environ["CSM_TARGET_SMTP_SERVER_ADDRESS"] = ( - rsu_error_summary_data.DEFAULT_CSM_TARGET_SMTP_SERVER_ADDRESS - ) - os.environ["CSM_TARGET_SMTP_SERVER_PORT"] = str( - rsu_error_summary_data.DEFAULT_CSM_TARGET_SMTP_SERVER_PORT - ) RSUErrorSummaryResource = rsu_error_summary.RSUErrorSummaryResource() # execute @@ -147,11 +175,4 @@ def test_validate_input_good(): ) # assert - assert result == None - - # cleanup - del os.environ["CSM_EMAIL_TO_SEND_FROM"] - del os.environ["CSM_EMAIL_APP_USERNAME"] - del os.environ["CSM_EMAIL_APP_PASSWORD"] - del os.environ["CSM_TARGET_SMTP_SERVER_ADDRESS"] - del os.environ["CSM_TARGET_SMTP_SERVER_PORT"] + assert result is None diff --git a/services/api/tests/src/test_rsu_geo_msg_query.py b/services/api/tests/src/test_rsu_geo_msg_query.py index a98cd244d..263d209ac 100644 --- a/services/api/tests/src/test_rsu_geo_msg_query.py +++ b/services/api/tests/src/test_rsu_geo_msg_query.py @@ -1,8 +1,6 @@ from copy import deepcopy from datetime import datetime, timedelta from unittest.mock import patch, MagicMock -import os - import pytest from api.src.rsu_geo_msg_query import ( query_geo_data_mongo, @@ -10,7 +8,6 @@ get_collection, create_geo_filter, ) -import json import api.tests.data.rsu_geo_msg_query_data as rsu_geo_msg_query_data from werkzeug.exceptions import InternalServerError @@ -20,15 +17,6 @@ def test_geo_hash(): assert result is not None -@patch.dict( - os.environ, - { - "MONGO_DB_URI": "uri", - "MONGO_DB_NAME": "name", - "MONGO_PROCESSED_BSM_COLLECTION_NAME": "col", - "MAX_GEO_QUERY_RECORDS": "10000", - }, -) @patch("api.src.rsu_geo_msg_query.MongoClient") def test_query_geo_data_mongo_bsm(mock_mongo): mock_db = MagicMock() @@ -57,15 +45,6 @@ def test_query_geo_data_mongo_bsm(mock_mongo): assert response == expected_response -@patch.dict( - os.environ, - { - "MONGO_DB_URI": "uri", - "MONGO_DB_NAME": "name", - "MONGO_PROCESSED_BSM_COLLECTION_NAME": "col", - "MAX_GEO_QUERY_RECORDS": "10000", - }, -) @patch("api.src.rsu_geo_msg_query.MongoClient") def test_query_geo_data_mongo_filter_failed(mock_mongo): mock_db = MagicMock() @@ -90,14 +69,6 @@ def test_query_geo_data_mongo_filter_failed(mock_mongo): mock_collection.find.assert_called() -@patch.dict( - os.environ, - { - "MONGO_DB_URI": "uri", - "MONGO_DB_NAME": "name", - "MONGO_PROCESSED_BSM_COLLECTION_NAME": "col", - }, -) @patch("api.src.rsu_geo_msg_query.MongoClient") def test_query_geo_data_mongo_failed_to_connect(mock_mongo): mock_mongo.side_effect = Exception("Failed to connect") @@ -114,15 +85,6 @@ def test_query_geo_data_mongo_failed_to_connect(mock_mongo): assert response == expected_response -@patch.dict( - os.environ, - { - "MONGO_DB_URI": "uri", - "MONGO_DB_NAME": "name", - "MONGO_PROCESSED_PSM_COLLECTION_NAME": "col", - "MAX_GEO_QUERY_RECORDS": "10000", - }, -) @patch("api.src.rsu_geo_msg_query.MongoClient") def test_query_geo_data_mongo_psm(mock_mongo): mock_db = MagicMock() @@ -161,14 +123,6 @@ def test_query_geo_data_mongo_psm(mock_mongo): # mock_collection.find.assert_called() -@patch.dict( - os.environ, - { - "MONGO_DB_URI": "uri", - "MONGO_DB_NAME": "name", - "MAX_GEO_QUERY_RECORDS": "10000", - }, -) @patch("api.src.rsu_geo_msg_query.MongoClient") def test_query_geo_data_mongo_unsupported_msg_type(mock_mongo): start = "2023-07-01T00:00:00Z" @@ -181,15 +135,8 @@ def test_query_geo_data_mongo_unsupported_msg_type(mock_mongo): assert response == [] -@patch.dict( - os.environ, - { - "MONGO_DB_URI": "uri", - "MONGO_DB_NAME": "name", - "MONGO_PROCESSED_BSM_COLLECTION_NAME": "bsm_col", - "MONGO_PROCESSED_PSM_COLLECTION_NAME": "psm_col", - }, -) +@patch("api_environment.MONGO_PROCESSED_BSM_COLLECTION_NAME", "bsm_col") +@patch("api_environment.MONGO_PROCESSED_PSM_COLLECTION_NAME", "psm_col") @patch("api.src.rsu_geo_msg_query.MongoClient") def test_get_collection(mock_mongo): mock_db = MagicMock() @@ -236,15 +183,6 @@ def test_create_geo_filter(): assert filter["geometry"]["$geoWithin"]["$geometry"]["coordinates"] == [point_list] -@patch.dict( - os.environ, - { - "MONGO_DB_URI": "uri", - "MONGO_DB_NAME": "name", - "MONGO_PROCESSED_BSM_COLLECTION_NAME": "col", - "MAX_GEO_QUERY_RECORDS": "10000", - }, -) @patch("api.src.rsu_geo_msg_query.MongoClient") def test_query_geo_data_mongo_schema_version_filter(mock_mongo): mock_db = MagicMock() @@ -270,15 +208,6 @@ def test_query_geo_data_mongo_schema_version_filter(mock_mongo): # checks deduplication and sorting by timestamp -@patch.dict( - os.environ, - { - "MONGO_DB_URI": "uri", - "MONGO_DB_NAME": "name", - "MONGO_PROCESSED_BSM_COLLECTION_NAME": "col", - "MAX_GEO_QUERY_RECORDS": "10000", - }, -) @patch("api.src.rsu_geo_msg_query.MongoClient") def test_order_by_time_stamp(mock_mongo): mock_db = MagicMock() @@ -302,15 +231,8 @@ def test_order_by_time_stamp(mock_mongo): ) -@patch.dict( - os.environ, - { - "MONGO_DB_URI": "uri", - "MONGO_DB_NAME": "name", - "MONGO_PROCESSED_BSM_COLLECTION_NAME": "col", - "MAX_GEO_QUERY_RECORDS": "5", - }, -) +@patch("api_environment.MONGO_PROCESSED_BSM_COLLECTION_NAME", "col") +@patch("api_environment.MAX_GEO_QUERY_RECORDS", 5) @patch("api.src.rsu_geo_msg_query.MongoClient") def test_query_limit(mock_mongo): mock_db = MagicMock() diff --git a/services/api/tests/src/test_rsu_online_status.py b/services/api/tests/src/test_rsu_online_status.py index 0e4e75983..4d5e28967 100644 --- a/services/api/tests/src/test_rsu_online_status.py +++ b/services/api/tests/src/test_rsu_online_status.py @@ -90,7 +90,10 @@ def test_ping_data_query(mock_pgquery): } rsu_online_status.get_ping_data(user_valid_no_super) - mock_pgquery.query_db.assert_called_with(expected_query, params=expected_params) + mock_pgquery.query_db.assert_called_with( + expected_query, + params=expected_params, + ) @patch("api.src.rsu_online_status.pgquery") diff --git a/services/api/tests/src/test_rsu_querycounts.py b/services/api/tests/src/test_rsu_querycounts.py index 3df026a0f..9c7a29c01 100644 --- a/services/api/tests/src/test_rsu_querycounts.py +++ b/services/api/tests/src/test_rsu_querycounts.py @@ -1,11 +1,10 @@ from unittest.mock import patch, MagicMock import pytest -import os import api.src.rsu_querycounts as rsu_querycounts -from api.src.rsu_querycounts import query_rsu_counts_mongo -import api.tests.data.rsu_querycounts_data as querycounts_data +from api.src.rsu_querycounts import query_rsu_counts_aggregated from api.tests.data import auth_data -from werkzeug.exceptions import BadRequest, Forbidden +from werkzeug.exceptions import Forbidden, InternalServerError +from datetime import datetime user_valid = auth_data.get_request_environ() @@ -19,169 +18,236 @@ def test_options_request(): assert headers["Access-Control-Allow-Methods"] == "GET" -@patch("api.src.rsu_querycounts.get_organization_rsus") -@patch("api.src.rsu_querycounts.query_rsu_counts_mongo") -@patch( - "api.src.rsu_querycounts.request", - MagicMock( - args=querycounts_data.request_args_good, - ), -) -def test_get_request(mock_query, mock_rsus): - counts = rsu_querycounts.RsuQueryCounts() - mock_rsus.return_value = ["10.0.0.1", "10.0.0.2", "10.0.0.3"] - mock_query.return_value = {"Some Data"} - (data, code, headers) = counts.get() - assert code == 200 - assert headers["Access-Control-Allow-Origin"] == "test.com" - assert headers["Content-Type"] == "application/json" - assert data == {"Some Data"} - - -# ################################## Testing Data Validation ######################################### -@patch.dict(os.environ, {"COUNTS_MSG_TYPES": "test,anothErtest"}) -@patch( - "api.src.rsu_querycounts.request", - MagicMock( - args=querycounts_data.request_args_bad_message, - ), -) -def test_get_request_invalid_message(): - counts = rsu_querycounts.RsuQueryCounts() +##################################### Test query_rsu_counts_aggregated ########################################### +@patch("api.src.rsu_querycounts.MongoClient") +@patch("api.src.rsu_querycounts.util.format_date_utc") +def test_query_rsu_counts_aggregated_success(mock_format_date, mock_mongo): + # Mock date formatting + mock_start_dt = datetime(2022, 1, 1, 0, 0, 0) + mock_end_dt = datetime(2023, 1, 1, 0, 0, 0) + mock_format_date.side_effect = [mock_start_dt, mock_end_dt] + + # Mock MongoDB connection and collection + mock_db = MagicMock() + mock_collection = MagicMock() + mock_mongo.return_value.__getitem__.return_value = mock_db + mock_db.__getitem__.return_value = mock_collection - with pytest.raises(BadRequest) as exc_info: - counts.get() + # Mock aggregation cursor result + mock_cursor = [ + { + "rsuIp": "192.168.0.1", + "messageTypeCounts": {"BSM": 100, "SPAT": 50, "SSM": 25}, + }, + { + "rsuIp": "192.168.0.2", + "messageTypeCounts": {"BSM": 200, "SPAT": 75}, + }, + ] + mock_collection.aggregate.return_value = iter(mock_cursor) - assert ( - str(exc_info.value) - == "400 Bad Request: Invalid Message Type.\nValid message types: Test, Anothertest" - ) + allowed_ips_dict = { + "192.168.0.1": "Route 1", + "192.168.0.2": "Route 2", + "192.168.0.3": "Route 3", + } + start = "2022-01-01T00:00:00" + end = "2023-01-01T00:00:00" + expected_result = { + "192.168.0.1": { + "road": "Route 1", + "messageTypeCounts": {"BSM": 100, "SPAT": 50, "SSM": 25}, + }, + "192.168.0.2": { + "road": "Route 2", + "messageTypeCounts": {"BSM": 200, "SPAT": 75}, + }, + "192.168.0.3": {"road": "Route 3", "messageTypeCounts": {}}, + } -@patch.dict(os.environ, {}, clear=True) -@patch( - "api.src.rsu_querycounts.request", - MagicMock( - args=querycounts_data.request_args_bad_message, - ), -) -def test_get_request_invalid_message_no_env(): - counts = rsu_querycounts.RsuQueryCounts() + result = query_rsu_counts_aggregated(allowed_ips_dict, start, end) - with pytest.raises(BadRequest) as exc_info: - counts.get() + assert result == expected_result + # Verify the aggregation pipeline was called with allowDiskUse=False + mock_collection.aggregate.assert_called_once() + call_args = mock_collection.aggregate.call_args + assert call_args[1]["allowDiskUse"] is False - assert ( - str(exc_info.value) - == "400 Bad Request: Invalid Message Type.\nValid message types: Bsm, Ssm, Spat, Srm, Map" - ) +@patch("api.src.rsu_querycounts.MongoClient") +@patch("api.src.rsu_querycounts.util.format_date_utc") +def test_query_rsu_counts_aggregated_empty_results(mock_format_date, mock_mongo): + # Mock date formatting + mock_start_dt = datetime(2022, 1, 1, 0, 0, 0) + mock_end_dt = datetime(2023, 1, 1, 0, 0, 0) + mock_format_date.side_effect = [mock_start_dt, mock_end_dt] + + # Mock MongoDB connection and collection + mock_db = MagicMock() + mock_collection = MagicMock() + mock_mongo.return_value.__getitem__.return_value = mock_db + mock_db.__getitem__.return_value = mock_collection -@patch( - "api.src.rsu_querycounts.request", - MagicMock( - args=querycounts_data.request_args_bad_type, - ), -) -def test_schema_validate_bad_data(): - counts = rsu_querycounts.RsuQueryCounts() - with pytest.raises(Exception): - assert counts.get() + # Mock empty aggregation cursor result + mock_collection.aggregate.return_value = iter([]) + allowed_ips_dict = { + "192.168.0.1": "Route 1", + "192.168.0.2": "Route 2", + } + start = "2022-01-01T00:00:00" + end = "2023-01-01T00:00:00" -# ################################## Test get_organization_rsus ######################################## -@patch("api.src.rsu_querycounts.pgquery") -def test_rsu_counts_get_organization_rsus(mock_pgquery): - mock_pgquery.query_db.return_value = [ - ({"ipv4_address": "10.11.81.12", "primary_route": "Route 1"},), - ({"ipv4_address": "10.11.81.13", "primary_route": "Route 1"},), - ({"ipv4_address": "10.11.81.14", "primary_route": "Route 1"},), - ] - expected_query = ( - "SELECT to_jsonb(row) " - "FROM (" - "SELECT rd.ipv4_address, rd.primary_route " - "FROM public.rsus rd " - "JOIN public.rsu_organization_name AS ron_v ON ron_v.rsu_id = rd.rsu_id " - "ORDER BY primary_route ASC, milepost ASC " - ") as row" - ) - - actual_result = rsu_querycounts.get_organization_rsus(user_valid, []) - - mock_pgquery.query_db.assert_called_with(expected_query, params={}) - assert actual_result == { - "10.11.81.12": "Route 1", - "10.11.81.13": "Route 1", - "10.11.81.14": "Route 1", + expected_result = { + "192.168.0.1": {"road": "Route 1", "messageTypeCounts": {}}, + "192.168.0.2": {"road": "Route 2", "messageTypeCounts": {}}, } + result = query_rsu_counts_aggregated(allowed_ips_dict, start, end) + + assert result == expected_result + -@patch("api.src.rsu_querycounts.pgquery") -def test_rsu_counts_get_organization_rsus_empty(mock_pgquery): - mock_pgquery.query_db.return_value = [] - expected_query = ( - "SELECT to_jsonb(row) " - "FROM (" - "SELECT rd.ipv4_address, rd.primary_route " - "FROM public.rsus rd " - "JOIN public.rsu_organization_name AS ron_v ON ron_v.rsu_id = rd.rsu_id " - "ORDER BY primary_route ASC, milepost ASC " - ") as row" - ) - actual_result = rsu_querycounts.get_organization_rsus(user_valid, []) - mock_pgquery.query_db.assert_called_with(expected_query, params={}) - - assert actual_result == {} - - -##################################### Test query_rsu_counts ########################################### -@patch.dict( - os.environ, - {"MONGO_DB_URI": "uri", "MONGO_DB_NAME": "name"}, -) @patch("api.src.rsu_querycounts.MongoClient") -def test_query_rsu_counts_mongo_success(mock_mongo): +@patch("api.src.rsu_querycounts.logging") +def test_query_rsu_counts_aggregated_connection_failure(mock_logging, mock_mongo): + # Mock the MongoDB connection to throw an exception + mock_mongo.side_effect = Exception("Connection timeout") + + allowed_ips_dict = {"192.168.0.1": "Route 1"} + start = "2022-01-01T00:00:00" + end = "2023-01-01T00:00:00" + + with pytest.raises(Forbidden) as exc_info: + query_rsu_counts_aggregated(allowed_ips_dict, start, end) + + assert str(exc_info.value) == "403 Forbidden: Failed to connect to Mongo" + mock_logging.error.assert_called_once() + + +@patch("api.src.rsu_querycounts.MongoClient") +@patch("api.src.rsu_querycounts.util.format_date_utc") +@patch("api.src.rsu_querycounts.logging") +def test_query_rsu_counts_aggregated_aggregation_failure( + mock_logging, mock_format_date, mock_mongo +): + # Mock date formatting + mock_start_dt = datetime(2022, 1, 1, 0, 0, 0) + mock_end_dt = datetime(2023, 1, 1, 0, 0, 0) + mock_format_date.side_effect = [mock_start_dt, mock_end_dt] + + # Mock MongoDB connection and collection mock_db = MagicMock() mock_collection = MagicMock() mock_mongo.return_value.__getitem__.return_value = mock_db mock_db.__getitem__.return_value = mock_collection - mock_db.validate_collection.return_value = "valid" - # Mock data that would be returned from MongoDB - mock_collection.find_one.return_value = {"count": 5} + # Mock aggregation to throw an exception + mock_collection.aggregate.side_effect = Exception("Aggregation error") - allowed_ips = {"192.168.0.1": "A1", "192.168.0.2": "A2"} - message_type = "BSM" + allowed_ips_dict = {"192.168.0.1": "Route 1"} + start = "2022-01-01T00:00:00" + end = "2023-01-01T00:00:00" + + with pytest.raises(InternalServerError) as exc_info: + query_rsu_counts_aggregated(allowed_ips_dict, start, end) + + assert str(exc_info.value) == "500 Internal Server Error: Encountered unknown issue" + mock_logging.error.assert_called() + + +@patch("api.src.rsu_querycounts.MongoClient") +@patch("api.src.rsu_querycounts.util.format_date_utc") +def test_query_rsu_counts_aggregated_partial_results(mock_format_date, mock_mongo): + # Mock date formatting + mock_start_dt = datetime(2022, 1, 1, 0, 0, 0) + mock_end_dt = datetime(2023, 1, 1, 0, 0, 0) + mock_format_date.side_effect = [mock_start_dt, mock_end_dt] + + # Mock MongoDB connection and collection + mock_db = MagicMock() + mock_collection = MagicMock() + mock_mongo.return_value.__getitem__.return_value = mock_db + mock_db.__getitem__.return_value = mock_collection + + # Mock aggregation cursor with only one RSU having results + mock_cursor = [ + { + "rsuIp": "192.168.0.1", + "messageTypeCounts": {"BSM": 150}, + } + ] + mock_collection.aggregate.return_value = iter(mock_cursor) + + allowed_ips_dict = { + "192.168.0.1": "Route 1", + "192.168.0.2": "Route 2", + "192.168.0.3": "Route 3", + } start = "2022-01-01T00:00:00" end = "2023-01-01T00:00:00" expected_result = { - "192.168.0.1": {"road": "A1", "count": 5}, - "192.168.0.2": {"road": "A2", "count": 5}, + "192.168.0.1": {"road": "Route 1", "messageTypeCounts": {"BSM": 150}}, + "192.168.0.2": {"road": "Route 2", "messageTypeCounts": {}}, + "192.168.0.3": {"road": "Route 3", "messageTypeCounts": {}}, } - result = query_rsu_counts_mongo(allowed_ips, message_type, start, end) + result = query_rsu_counts_aggregated(allowed_ips_dict, start, end) assert result == expected_result -@patch.dict( - os.environ, - {"MONGO_DB_URI": "uri", "MONGO_DB_NAME": "name"}, -) @patch("api.src.rsu_querycounts.MongoClient") -@patch("api.src.rsu_querycounts.logging") -def test_query_rsu_counts_mongo_failure(mock_logging, mock_mongo): - # Mock the MongoDB connection to throw an exception - mock_mongo.side_effect = Exception("Failed to connect") +@patch("api.src.rsu_querycounts.util.format_date_utc") +def test_query_rsu_counts_aggregated_pipeline_structure(mock_format_date, mock_mongo): + # Mock date formatting + mock_start_dt = datetime(2022, 1, 1, 0, 0, 0) + mock_end_dt = datetime(2023, 1, 1, 0, 0, 0) + mock_format_date.side_effect = [mock_start_dt, mock_end_dt] + + # Mock MongoDB connection and collection + mock_db = MagicMock() + mock_collection = MagicMock() + mock_mongo.return_value.__getitem__.return_value = mock_db + mock_db.__getitem__.return_value = mock_collection + mock_collection.aggregate.return_value = iter([]) - allowed_ips = ["192.168.0.1", "192.168.0.2"] - message_type = "TYPE_A" + allowed_ips_dict = {"192.168.0.1": "Route 1", "192.168.0.2": "Route 2"} start = "2022-01-01T00:00:00" end = "2023-01-01T00:00:00" - with pytest.raises(Forbidden) as exc_info: - query_rsu_counts_mongo(allowed_ips, message_type, start, end) + query_rsu_counts_aggregated(allowed_ips_dict, start, end) + + # Verify the pipeline structure + call_args = mock_collection.aggregate.call_args + pipeline = call_args[0][0] + + # Check match stage + assert pipeline[0]["$match"]["rsuIp"]["$in"] == ["192.168.0.1", "192.168.0.2"] + assert pipeline[0]["$match"]["timestamp"]["$gte"] == mock_start_dt + assert pipeline[0]["$match"]["timestamp"]["$lt"] == mock_end_dt + + # Check project stage + assert "$project" in pipeline[1] + assert pipeline[1]["$project"]["rsuIp"] == 1 + assert pipeline[1]["$project"]["messageType"] == 1 + assert pipeline[1]["$project"]["count"] == 1 + assert pipeline[1]["$project"]["_id"] == 0 + + # Check first group stage + assert "$group" in pipeline[2] + assert pipeline[2]["$group"]["_id"] == { + "rsuIp": "$rsuIp", + "messageType": "$messageType", + } - assert str(exc_info.value) == "403 Forbidden: Failed to connect to Mongo" + # Check second group stage + assert "$group" in pipeline[3] + assert pipeline[3]["$group"]["_id"] == "$_id.rsuIp" + + # Check final project stage + assert "$project" in pipeline[4] + assert pipeline[4]["$project"]["_id"] == 0 + assert pipeline[4]["$project"]["rsuIp"] == "$_id" diff --git a/services/api/tests/src/test_rsu_ssm_srm.py b/services/api/tests/src/test_rsu_ssm_srm.py index b16d5d704..ba2c5eea4 100644 --- a/services/api/tests/src/test_rsu_ssm_srm.py +++ b/services/api/tests/src/test_rsu_ssm_srm.py @@ -49,15 +49,7 @@ def test_get_request(mock_get_rsu_dict, mock_srm, mock_ssm): ] -@patch.dict( - os.environ, - { - "MONGO_DB_URI": "uri", - "MONGO_DB_NAME": "db", - "SSM_DB_NAME": "collection", - "SRM_DB_NAME": "srm_collection", - }, -) +@patch("api_environment.MONGO_SSM_COLLECTION_NAME", "ssm_collection") @patch("api.src.rsu_ssm_srm.query_ssm_data_mongo") @patch("api.src.rsu_ssm_srm.query_srm_data_mongo") @patch("api.src.rsu_ssm_srm.get_rsu_set_for_org") @@ -79,10 +71,7 @@ def test_get_request_invalid(mock_get_rsu_dict, mock_srm, mock_ssm): # ################################### Test query_ssm_data ######################################## -@patch.dict( - os.environ, - {"MONGO_DB_URI": "uri", "MONGO_DB_NAME": "db", "SSM_DB_NAME": "collection"}, -) +@patch("api_environment.MONGO_SSM_COLLECTION_NAME", "ssm_collection") @patch("api.src.rsu_ssm_srm.MongoClient") @patch("api.src.rsu_ssm_srm.datetime") def test_query_ssm_data_query(mock_date, mock_mongo): @@ -103,10 +92,7 @@ def test_query_ssm_data_query(mock_date, mock_mongo): mock_collection.find.assert_called() -@patch.dict( - os.environ, - {"MONGO_DB_URI": "uri", "MONGO_DB_NAME": "db", "SSM_DB_NAME": "collection"}, -) +@patch("api_environment.MONGO_SSM_COLLECTION_NAME", "Fake_table") @patch("api.src.rsu_ssm_srm.MongoClient") def test_query_ssm_data_no_data(mock_mongo): mock_db = MagicMock() @@ -115,15 +101,11 @@ def test_query_ssm_data_no_data(mock_mongo): mock_db.__getitem__.return_value = mock_collection mock_collection.find.return_value = [] - with patch.dict("api.src.rsu_ssm_srm.os.environ", {"SSM_DB_NAME": "Fake_table"}): - data = rsu_ssm_srm.query_ssm_data_mongo() - assert data == [] + data = rsu_ssm_srm.query_ssm_data_mongo() + assert data == [] -@patch.dict( - os.environ, - {"MONGO_DB_URI": "uri", "MONGO_DB_NAME": "db", "SSM_DB_NAME": "collection"}, -) +@patch("api_environment.MONGO_SRM_COLLECTION_NAME", "Fake_table") @patch("api.src.rsu_ssm_srm.MongoClient") def test_query_ssm_data_single_result(mock_mongo): mock_db = MagicMock() @@ -132,9 +114,8 @@ def test_query_ssm_data_single_result(mock_mongo): mock_db.__getitem__.return_value = mock_collection mock_collection.find.return_value = [ssm_srm_data.ssm_record_one] - with patch.dict("api.src.rsu_ssm_srm.os.environ", {"SSM_DB_NAME": "Fake_table"}): - data = rsu_ssm_srm.query_ssm_data_mongo() - assert data == ssm_srm_data.ssm_single_result_expected + data = rsu_ssm_srm.query_ssm_data_mongo() + assert data == ssm_srm_data.ssm_single_result_expected @patch.dict( @@ -153,16 +134,12 @@ def test_query_ssm_data_multiple_result(mock_mongo): ssm_srm_data.ssm_record_two, ssm_srm_data.ssm_record_three, ] - with patch.dict("api.src.rsu_ssm_srm.os.environ", {"SSM_DB_NAME": "Fake_table"}): - data = rsu_ssm_srm.query_ssm_data_mongo() - assert data == ssm_srm_data.ssm_multiple_result_expected + data = rsu_ssm_srm.query_ssm_data_mongo() + assert data == ssm_srm_data.ssm_multiple_result_expected # #################################### Test query_srm_data ########################################### -@patch.dict( - os.environ, - {"MONGO_DB_URI": "uri", "MONGO_DB_NAME": "db", "SRM_DB_NAME": "collection"}, -) +@patch("api_environment.MONGO_SRM_COLLECTION_NAME", "Fake_table") @patch("api.src.rsu_ssm_srm.MongoClient") @patch("api.src.rsu_ssm_srm.datetime") def test_query_srm_data_query(mock_date, mock_mongo): @@ -175,16 +152,12 @@ def test_query_srm_data_query(mock_date, mock_mongo): mock_date.now.return_value = datetime.strptime( "2022/12/14 00:00:00", "%Y/%m/%d %H:%M:%S" ).astimezone(UTC) - with patch.dict("api.src.rsu_ssm_srm.os.environ", {"SRM_DB_NAME": "Fake_table"}): - rsu_ssm_srm.query_srm_data_mongo() - mock_mongo.assert_called() - mock_collection.find.assert_called() + rsu_ssm_srm.query_srm_data_mongo() + mock_mongo.assert_called() + mock_collection.find.assert_called() -@patch.dict( - os.environ, - {"MONGO_DB_URI": "uri", "MONGO_DB_NAME": "db", "SRM_DB_NAME": "collection"}, -) +@patch("api_environment.MONGO_SRM_COLLECTION_NAME", "ssm_collection") @patch("api.src.rsu_ssm_srm.MongoClient") def test_query_srm_data_no_data(mock_mongo): mock_db = MagicMock() @@ -193,15 +166,11 @@ def test_query_srm_data_no_data(mock_mongo): mock_db.__getitem__.return_value = mock_collection mock_collection.find.return_value = [] - with patch.dict("api.src.rsu_ssm_srm.os.environ", {"SRM_DB_NAME": "Fake_table"}): - data = rsu_ssm_srm.query_srm_data_mongo() - assert data == [] + data = rsu_ssm_srm.query_srm_data_mongo() + assert data == [] -@patch.dict( - os.environ, - {"MONGO_DB_URI": "uri", "MONGO_DB_NAME": "db", "SRM_DB_NAME": "collection"}, -) +@patch("api_environment.MONGO_SRM_COLLECTION_NAME", "Fake_table") @patch("api.src.rsu_ssm_srm.MongoClient") def test_query_srm_data_single_result(mock_mongo): mock_db = MagicMock() @@ -210,15 +179,11 @@ def test_query_srm_data_single_result(mock_mongo): mock_db.__getitem__.return_value = mock_collection mock_collection.find.return_value = [ssm_srm_data.srm_record_one] - with patch.dict("api.src.rsu_ssm_srm.os.environ", {"SRM_DB_NAME": "Fake_table"}): - data = rsu_ssm_srm.query_srm_data_mongo() - assert data == ssm_srm_data.srm_single_result_expected + data = rsu_ssm_srm.query_srm_data_mongo() + assert data == ssm_srm_data.srm_single_result_expected -@patch.dict( - os.environ, - {"MONGO_DB_URI": "uri", "MONGO_DB_NAME": "db", "SRM_DB_NAME": "collection"}, -) +@patch("api_environment.MONGO_SRM_COLLECTION_NAME", "Fake_table") @patch("api.src.rsu_ssm_srm.MongoClient") def test_query_srm_data_multiple_result(mock_mongo): mock_db = MagicMock() @@ -231,6 +196,5 @@ def test_query_srm_data_multiple_result(mock_mongo): ssm_srm_data.srm_record_two, ssm_srm_data.srm_record_three, ] - with patch.dict("api.src.rsu_ssm_srm.os.environ", {"SRM_DB_NAME": "Fake_table"}): - data = rsu_ssm_srm.query_srm_data_mongo() - assert data == ssm_srm_data.srm_multiple_result_expected + data = rsu_ssm_srm.query_srm_data_mongo() + assert data == ssm_srm_data.srm_multiple_result_expected diff --git a/services/api/tests/src/test_rsu_upgrade.py b/services/api/tests/src/test_rsu_upgrade.py index 8331d97d1..3e2a489b0 100644 --- a/services/api/tests/src/test_rsu_upgrade.py +++ b/services/api/tests/src/test_rsu_upgrade.py @@ -2,7 +2,6 @@ import pytest from api.src import rsu_upgrade -import os from werkzeug.exceptions import Conflict @@ -77,7 +76,7 @@ def test_check_for_upgrade_false(mock_query_db): assert actual_response == expected_response -@patch.dict(os.environ, {"FIRMWARE_MANAGER_ENDPOINT": "http://1.1.1.1:8080"}) +@patch("api_environment.FIRMWARE_MANAGER_ENDPOINT", "http://1.1.1.1:8080") @patch("api.src.rsu_upgrade.requests.post") @patch("api.src.rsu_upgrade.pgquery.write_db") @patch( @@ -116,7 +115,7 @@ def test_mark_rsu_for_upgrade_eligible( assert actual_status_code == expected_status_code -@patch.dict(os.environ, {"FIRMWARE_MANAGER_ENDPOINT": "http://1.1.1.1:8080"}) +@patch("api_environment.FIRMWARE_MANAGER_ENDPOINT", "http://1.1.1.1:8080") @patch("api.src.rsu_upgrade.requests.post") @patch("api.src.rsu_upgrade.pgquery.write_db") @patch( @@ -157,7 +156,7 @@ def test_mark_rsu_for_upgrade_eligible_but_rejected( assert actual_status_code == expected_status_code -@patch.dict(os.environ, {"FIRMWARE_MANAGER_ENDPOINT": "http://1.1.1.1:8080"}) +@patch("api_environment.FIRMWARE_MANAGER_ENDPOINT", "http://1.1.1.1:8080") @patch("api.src.rsu_upgrade.requests.post") @patch("api.src.rsu_upgrade.pgquery.write_db") @patch( diff --git a/services/api/tests/src/test_smtp_error_handler.py b/services/api/tests/src/test_smtp_error_handler.py index 039233b18..490fe39c7 100644 --- a/services/api/tests/src/test_smtp_error_handler.py +++ b/services/api/tests/src/test_smtp_error_handler.py @@ -1,6 +1,6 @@ from collections import namedtuple -import os -from unittest.mock import patch, MagicMock, call, mock_open +import logging +from unittest.mock import patch, MagicMock, mock_open from api.src.smtp_error_handler import SMTP_SSLHandler import api.src.smtp_error_handler as smtp_error_handler import api.tests.data.smtp_error_handler_data as smtp_error_handler_data @@ -22,10 +22,9 @@ def test_get_environment_name_fail(): ###################################### Testing Functions ########################################## -@patch.dict( - os.environ, - {"CSM_EMAILS_TO_SEND_TO": "test@gmail.com,test2@gmail.com"}, - clear=True, +@patch( + "api_environment.CSM_EMAILS_TO_SEND_TO", + ["test@gmail.com", "test2@gmail.com"], ) def test_get_subscribed_users_success(): expected = ["test@gmail.com", "test2@gmail.com"] @@ -34,26 +33,26 @@ def test_get_subscribed_users_success(): EMAIL_TO_SEND_FROM = "test@test.test" +EMAILS_TO_SEND_TO = ["test1@gmail.com", "test2@gmail.com"] EMAIL_APP_USERNAME = "test" EMAIL_APP_PASSWORD = "test" DEFAULT_TARGET_SMTP_SERVER_ADDRESS = "smtp.gmail.com" DEFAULT_TARGET_SMTP_SERVER_PORT = 587 ENVIRONMENT_NAME = "ENVIRONMENT" LOGS_LINK = "http://logs_link.com" -ERROR_EMAIL_UNSUBSCRIBE_LINK = "http://unsubscribe-{email}" - - -@patch.dict( - os.environ, - { - "CSM_TARGET_SMTP_SERVER_ADDRESS": DEFAULT_TARGET_SMTP_SERVER_ADDRESS, - "CSM_TARGET_SMTP_SERVER_PORT": str(DEFAULT_TARGET_SMTP_SERVER_PORT), - "CSM_EMAIL_TO_SEND_FROM": EMAIL_TO_SEND_FROM, - "CSM_EMAIL_APP_USERNAME": EMAIL_APP_USERNAME, - "CSM_EMAIL_APP_PASSWORD": EMAIL_APP_PASSWORD, - }, - clear=True, + + +@patch( + "api_environment.CSM_TARGET_SMTP_SERVER_ADDRESS", + DEFAULT_TARGET_SMTP_SERVER_ADDRESS, +) +@patch( + "api_environment.CSM_TARGET_SMTP_SERVER_PORT", + DEFAULT_TARGET_SMTP_SERVER_PORT, ) +@patch("api_environment.CSM_EMAIL_TO_SEND_FROM", EMAIL_TO_SEND_FROM) +@patch("api_environment.CSM_EMAIL_APP_USERNAME", EMAIL_APP_USERNAME) +@patch("api_environment.CSM_EMAIL_APP_PASSWORD", EMAIL_APP_PASSWORD) def test_configure_error_emails(): app = MagicMock() app.logger = MagicMock() @@ -62,19 +61,12 @@ def test_configure_error_emails(): app.logger.addHandler.assert_called_once() -@patch.dict( - os.environ, - { - "LOGS_LINK": LOGS_LINK, - "ENVIRONMENT_NAME": ENVIRONMENT_NAME, - "ERROR_EMAIL_UNSUBSCRIBE_LINK": ERROR_EMAIL_UNSUBSCRIBE_LINK, - }, - clear=True, -) +@patch("api_environment.LOGS_LINK", LOGS_LINK) +@patch("api_environment.ENVIRONMENT_NAME", ENVIRONMENT_NAME) +@patch("api_environment.CSM_EMAILS_TO_SEND_TO", EMAILS_TO_SEND_TO) @patch("builtins.open", new_callable=mock_open, read_data="data") @patch("api.src.smtp_error_handler.smtplib") -@patch("api.src.smtp_error_handler.get_subscribed_users") -def test_send(mock_get_subscribed_users, mock_smtplib, mock_file): +def test_send(mock_smtplib, mock_file): # prepare emailHandler = SMTP_SSLHandler( mailhost=[DEFAULT_TARGET_SMTP_SERVER_ADDRESS, DEFAULT_TARGET_SMTP_SERVER_PORT], @@ -94,22 +86,20 @@ def test_send(mock_get_subscribed_users, mock_smtplib, mock_file): smtp_obj.login = MagicMock() smtp_obj.sendmail = MagicMock() smtp_obj.quit = MagicMock() - mock_get_subscribed_users.return_value = ( - smtp_error_handler_data.subscribed_user_emails - ) mock_file = MagicMock() mock_file.read = MagicMock() mock_file.read.return_value = smtp_error_handler_data.html_email_template Record = namedtuple("Record", ["asctime"]) record = Record("2023-09-15 00:00:00,000000") - emailHandler.format = lambda x: "%s".format(x) + emailHandler.format = lambda x: str(x) + + logging.error("mock_smtplib", mock_smtplib) # execute emailHandler.emit(record) # assert - mock_get_subscribed_users.assert_called_once() smtp_obj.starttls.assert_called_once() smtp_obj.ehlo.assert_called_once() smtp_obj.login.assert_called_once_with(EMAIL_APP_USERNAME, EMAIL_APP_PASSWORD) diff --git a/services/api/tests/src/test_wzdx_feed.py b/services/api/tests/src/test_wzdx_feed.py index 9a66311f6..6329fb511 100644 --- a/services/api/tests/src/test_wzdx_feed.py +++ b/services/api/tests/src/test_wzdx_feed.py @@ -1,8 +1,11 @@ from unittest.mock import MagicMock, Mock + +from mock import patch from api.src import wzdx_feed -import os +@patch("api_environment.WZDX_ENDPOINT", "myendpoint") +@patch("api_environment.WZDX_API_KEY", "myapikey") # test that get_wzdx_data is calling json.loads with expected arguments def test_get_wzdx_data(): # mock return values for function dependencies @@ -14,12 +17,6 @@ def test_get_wzdx_data(): ) ) - endpoint = "my_endpoint" - api_key = "my_api_key" - - os.environ["WZDX_ENDPOINT"] = endpoint - os.environ["WZDX_API_KEY"] = api_key - # call function result = wzdx_feed.get_wzdx_data() diff --git a/services/common/common_environment.py b/services/common/common_environment.py new file mode 100644 index 000000000..1639be5bf --- /dev/null +++ b/services/common/common_environment.py @@ -0,0 +1,41 @@ +import os +import logging + + +def configure_logging() -> str: + LOGGING_LEVEL = os.environ.get("LOGGING_LEVEL", "INFO") + if not LOGGING_LEVEL or LOGGING_LEVEL == "": + LOGGING_LEVEL = "INFO" + logging.basicConfig(format="%(levelname)s:%(message)s", level=LOGGING_LEVEL) + return LOGGING_LEVEL + + +def get_env_var(key: str, default: str | None = None, error=False, warn=True): + value = os.environ.get(key) + if value is None or value == "": + if error: + raise Exception(f"Missing required environment variable: {key}") + if warn: + logging.warning( + f"Missing environment variable: {key}, using default: {default}" + ) + else: + logging.info( + f"Environment variable {key} was not specified, using default: {default}" + ) + return default + logging.info(f"Environment variable {key} is set to {value}") + return value + + +LOGGING_LEVEL = configure_logging() +TIMEZONE = get_env_var("TIMEZONE", "America/Denver") + +GCP_PROJECT = get_env_var("GCP_PROJECT") +BLOB_STORAGE_BUCKET = get_env_var("BLOB_STORAGE_BUCKET") + +PG_DB_HOST = get_env_var("PG_DB_HOST") +PG_DB_USER = get_env_var("PG_DB_USER") +PG_DB_PASS = get_env_var("PG_DB_PASS") +PG_DB_NAME = get_env_var("PG_DB_NAME") +INSTANCE_CONNECTION_NAME = get_env_var("INSTANCE_CONNECTION_NAME", "").strip() diff --git a/services/common/emailSender.py b/services/common/emailSender.py index e1cbf4eab..0e61ad7e5 100644 --- a/services/common/emailSender.py +++ b/services/common/emailSender.py @@ -22,8 +22,8 @@ def send( username, password, pretty=False, - tlsEnabled="true", - authEnabled="true", + tlsEnabled=True, + authEnabled=True, ): try: # prepare email @@ -33,10 +33,10 @@ def send( else: toSend = self.format(recipient, subject, message, replyEmail) - if tlsEnabled == "true": + if tlsEnabled: self.server.starttls(context=self.context) # start TLS encryption self.server.ehlo() # say hello - if authEnabled == "true": + if authEnabled: self.server.login(username, password) # send email diff --git a/services/common/gcs_utils.py b/services/common/gcs_utils.py index d7f14bd94..443c3ca5c 100644 --- a/services/common/gcs_utils.py +++ b/services/common/gcs_utils.py @@ -1,9 +1,9 @@ from google.cloud import storage from common.util import validate_file_type import logging +from common import common_environment import os - def download_gcp_blob(blob_name, destination_file_name, file_extension=None): """Download a file from a GCP Bucket Storage bucket to a local file. @@ -18,8 +18,8 @@ def download_gcp_blob(blob_name, destination_file_name, file_extension=None): if not validate_file_type(blob_name, file_extension): return False - gcp_project = os.environ.get("GCP_PROJECT") - bucket_name = os.environ.get("BLOB_STORAGE_BUCKET") + gcp_project = common_environment.GCP_PROJECT + bucket_name = common_environment.BLOB_STORAGE_BUCKET storage_client = storage.Client(gcp_project) bucket = storage_client.get_bucket(bucket_name) blob = bucket.blob(blob_name) @@ -35,8 +35,8 @@ def download_gcp_blob(blob_name, destination_file_name, file_extension=None): def list_gcs_blobs(gcs_prefix, file_extension): files = [] - gcp_project = os.environ.get("GCP_PROJECT") - bucket_name = os.environ.get("BLOB_STORAGE_BUCKET") + gcp_project = common_environment.GCP_PROJECT + bucket_name = common_environment.BLOB_STORAGE_BUCKET logging.debug(f"Listing blobs in bucket {bucket_name} with prefix {gcs_prefix}.") storage_client = storage.Client(gcp_project) blobs = storage_client.list_blobs(bucket_name, prefix=gcs_prefix, delimiter="/") diff --git a/services/common/pgquery.py b/services/common/pgquery.py index 40c9e02f6..51e6d20f4 100644 --- a/services/common/pgquery.py +++ b/services/common/pgquery.py @@ -1,6 +1,6 @@ -import os import sqlalchemy import logging +from common import common_environment db_config = { # Pool size is the maximum number of permanent connections to keep. @@ -60,22 +60,18 @@ def init_socket_connection_engine(db_user, db_pass, db_name, unix_query): def init_connection_engine(): - db_user = os.environ["PG_DB_USER"] - db_pass = os.environ["PG_DB_PASS"] - db_name = os.environ["PG_DB_NAME"] - if ( - "INSTANCE_CONNECTION_NAME" in os.environ - and os.environ["INSTANCE_CONNECTION_NAME"].strip() - ): + db_user = common_environment.PG_DB_USER + db_pass = common_environment.PG_DB_PASS + db_name = common_environment.PG_DB_NAME + if common_environment.INSTANCE_CONNECTION_NAME: logging.debug("Using socket connection") - instance_connection_name = os.environ["INSTANCE_CONNECTION_NAME"] unix_query = { - "unix_sock": f"/cloudsql/{instance_connection_name}/.s.PGSQL.5432" + "unix_sock": f"/cloudsql/{common_environment.INSTANCE_CONNECTION_NAME}/.s.PGSQL.5432" } return init_socket_connection_engine(db_user, db_pass, db_name, unix_query) else: logging.debug("Using tcp connection") - db_host = os.environ["PG_DB_HOST"] + db_host = common_environment.PG_DB_HOST # Extract host and port from db_host host_args = db_host.split(":") db_hostname, db_port = host_args[0], int(host_args[1]) diff --git a/services/common/tests/test_emailSender.py b/services/common/tests/test_emailSender.py index 0677f5f27..b2e5a4ed7 100644 --- a/services/common/tests/test_emailSender.py +++ b/services/common/tests/test_emailSender.py @@ -35,8 +35,8 @@ def test_send_with_tls_and_auth(): EMAIL_APP_USERNAME, EMAIL_APP_PASSWORD, False, - "true", # tlsEnabled - "true", # authEnabled + True, # tlsEnabled + True, # authEnabled ) # assert @@ -69,8 +69,8 @@ def test_send_with_tls_and_no_auth(): EMAIL_APP_USERNAME, EMAIL_APP_PASSWORD, False, - "true", # tlsEnabled - "false", # authEnabled + True, # tlsEnabled + False, # authEnabled ) # assert @@ -103,8 +103,8 @@ def test_send_with_no_tls_and_auth(): EMAIL_APP_USERNAME, EMAIL_APP_PASSWORD, False, - "false", # tlsEnabled - "true", # authEnabled + False, # tlsEnabled + True, # authEnabled ) # assert @@ -137,8 +137,8 @@ def test_send_with_no_tls_and_no_auth(): EMAIL_APP_USERNAME, EMAIL_APP_PASSWORD, False, - "false", # tlsEnabled - "false", # authEnabled + False, # tlsEnabled + False, # authEnabled ) # assert @@ -146,4 +146,4 @@ def test_send_with_no_tls_and_no_auth(): emailSender.server.ehlo.assert_not_called() emailSender.server.login.assert_not_called() emailSender.server.sendmail.assert_called_once() - emailSender.server.quit.assert_called_once() \ No newline at end of file + emailSender.server.quit.assert_called_once() diff --git a/services/common/tests/test_gcs_utils.py b/services/common/tests/test_gcs_utils.py index d1da789fb..3f4bcfe9d 100644 --- a/services/common/tests/test_gcs_utils.py +++ b/services/common/tests/test_gcs_utils.py @@ -3,9 +3,8 @@ from common import gcs_utils -@patch.dict( - os.environ, {"GCP_PROJECT": "test-project", "BLOB_STORAGE_BUCKET": "test-bucket"} -) +@patch("common.common_environment.GCP_PROJECT", "test-project") +@patch("common.common_environment.BLOB_STORAGE_BUCKET", "test-bucket") @patch("common.gcs_utils.logging") @patch("common.gcs_utils.storage.Client") def test_download_gcp_blob(mock_storage_client, mock_logging): @@ -29,9 +28,8 @@ def test_download_gcp_blob(mock_storage_client, mock_logging): ) -@patch.dict( - os.environ, {"GCP_PROJECT": "test-project", "BLOB_STORAGE_BUCKET": "test-bucket"} -) +@patch("common.common_environment.GCP_PROJECT", "test-project") +@patch("common.common_environment.BLOB_STORAGE_BUCKET", "test-bucket") @patch("common.gcs_utils.storage.Client") def test_list_gcs_blobs(mock_storage_client): # mock @@ -58,9 +56,8 @@ def test_list_gcs_blobs(mock_storage_client): assert result == ["/firmwares/file1.txt", "/firmwares/file3.txt"] -@patch.dict( - os.environ, {"GCP_PROJECT": "test-project", "BLOB_STORAGE_BUCKET": "test-bucket"} -) +@patch("common.common_environment.GCP_PROJECT", "test-project") +@patch("common.common_environment.BLOB_STORAGE_BUCKET", "test-bucket") @patch("common.gcs_utils.storage.Client") def test_list_gcs_blobs_empty(mock_storage_client): # mock diff --git a/services/common/tests/test_pgquery.py b/services/common/tests/test_pgquery.py index ab3c40f05..db73cf591 100644 --- a/services/common/tests/test_pgquery.py +++ b/services/common/tests/test_pgquery.py @@ -1,7 +1,6 @@ from unittest.mock import MagicMock, patch, Mock from common import pgquery import sqlalchemy -import os # test that init_tcp_connection_engine is calling sqlalchemy.create_engine with expected arguments @@ -93,6 +92,10 @@ def test_init_socket_connection_engine(): "common.pgquery.db_config", new={"pool_size": 5, "max_overflow": 2, "pool_timeout": 30, "pool_recycle": 1800}, ) +@patch("common.common_environment.PG_DB_USER", "user") +@patch("common.common_environment.PG_DB_PASS", "pass") +@patch("common.common_environment.PG_DB_NAME", "my_database") +@patch("common.common_environment.PG_DB_HOST", "my_hostname:3000") def test_init_connection_engine_target_tcp(): # mock return values for function dependencies pgquery.init_tcp_connection_engine = MagicMock(return_value="my_engine1") @@ -103,12 +106,6 @@ def test_init_connection_engine_target_tcp(): db_name = "my_database" db_hostname = "my_hostname:3000" - # set environment variables - os.environ["PG_DB_USER"] = db_user - os.environ["PG_DB_PASS"] = db_pass - os.environ["PG_DB_NAME"] = db_name - os.environ["PG_DB_HOST"] = db_hostname - host_args = db_hostname.split(":") db_hostname, db_port = host_args[0], int(host_args[1]) @@ -133,6 +130,13 @@ def test_init_connection_engine_target_tcp(): new={"pool_size": 5, "max_overflow": 2, "pool_timeout": 30, "pool_recycle": 1800}, ) @patch("common.pgquery.db", new=None) +@patch("common.common_environment.PG_DB_USER", "user") +@patch("common.common_environment.PG_DB_PASS", "pass") +@patch("common.common_environment.PG_DB_NAME", "my_database") +@patch( + "common.common_environment.INSTANCE_CONNECTION_NAME", + "myproject:us-central1:myinstance", +) def test_init_connection_engine_target_socket(): # mock return values for function dependencies pgquery.init_tcp_connection_engine = MagicMock(return_value="my_engine1") @@ -142,14 +146,8 @@ def test_init_connection_engine_target_socket(): db_pass = "pass" db_name = "my_database" - # set environment variables - os.environ["PG_DB_USER"] = db_user - os.environ["PG_DB_PASS"] = db_pass - os.environ["PG_DB_NAME"] = db_name - os.environ["INSTANCE_CONNECTION_NAME"] = "my_project:us-central1:my_instance" - unix_query = { - "unix_sock": f"/cloudsql/{os.environ['INSTANCE_CONNECTION_NAME']}/.s.PGSQL.5432" + "unix_sock": "/cloudsql/myproject:us-central1:myinstance/.s.PGSQL.5432" } # call function diff --git a/services/common/util.py b/services/common/util.py index 92405210f..915becf1f 100644 --- a/services/common/util.py +++ b/services/common/util.py @@ -1,7 +1,8 @@ from dateutil.parser import parse import pytz -import os import logging +import datetime +from common import common_environment # expects datetime string @@ -24,7 +25,7 @@ def format_date_denver(d): if not d: return None tmp = parse(d) - denver_tz = tmp.astimezone(pytz.timezone(os.getenv("TIMEZONE", "America/Denver"))) + denver_tz = tmp.astimezone(pytz.timezone(common_environment.TIMEZONE)) return denver_tz.strftime("%m/%d/%Y %I:%M:%S %p") @@ -33,19 +34,19 @@ def format_date_denver_iso(d): if not d: return None tmp = parse(d) - denver_tz = tmp.astimezone(pytz.timezone(os.getenv("TIMEZONE", "America/Denver"))) + denver_tz = tmp.astimezone(pytz.timezone(common_environment.TIMEZONE)) return denver_tz.isoformat() # expects datetime, utilizes environment variable to custom timezone -def utc2tz(d): +def utc2tz(d: datetime.datetime): if not d: return None - tz_d = d.astimezone(pytz.timezone(os.getenv("TIMEZONE", "America/Denver"))) + tz_d = d.astimezone(pytz.timezone(common_environment.TIMEZONE)) return tz_d -def validate_file_type(file_name, extension=".tar"): +def validate_file_type(file_name: str, extension=".tar"): """Validate the file type of the file to be downloaded. Args: diff --git a/services/requirements.txt b/services/requirements.txt index f7e951deb..97f59f09f 100644 --- a/services/requirements.txt +++ b/services/requirements.txt @@ -23,7 +23,7 @@ pytz==2023.3.post1 Werkzeug==3.0.0 uuid==1.30 multidict==6.0.5 -python-keycloak==5.5.1 +python-keycloak==5.8.1 fabric==3.2.2 paramiko==3.5.0 scp==0.15.0 @@ -38,4 +38,5 @@ google-cloud-bigquery==3.29.0 pandas==2.2.3 shapely==2.0.7 db-dtypes==1.4.0 -PyJWT==2.10.1 \ No newline at end of file +trio==0.31.0 +PyJWT==2.10.1 diff --git a/webapp/.env.development b/webapp/.env.development index 9df5b87ba..067c68bda 100644 --- a/webapp/.env.development +++ b/webapp/.env.development @@ -1,25 +1,26 @@ # used to determine where to install build BUILD_PATH='./build' -# mapbox access token +############## REQUIRED ############## +DOCKER_HOST_IP= REACT_APP_MAPBOX_TOKEN= # used to swap endpoints for various environments or leave blank REACT_APP_ENV="dev" # base url for api -REACT_APP_GATEWAY_BASE_URL="http://cvmanager.local.com:8081" +REACT_APP_GATEWAY_BASE_URL="http://${DOCKER_HOST_IP}:8081" # base url or IP for keycloak -REACT_APP_KEYCLOAK_URL="http://cvmanager.auth.com:8084/" +REACT_APP_KEYCLOAK_URL="http://${DOCKER_HOST_IP}:8084/" REACT_APP_KEYCLOAK_REALM="cvmanager" REACT_APP_KEYCLOAK_CLIENT_ID=cvmanager-gui REACT_APP_COUNT_MESSAGE_TYPES='BSM,SSM,SPAT,SRM,MAP' REACT_APP_DOT_NAME="CDOT" - -REACT_APP_CVIZ_API_SERVER_URL='http://localhost:8089' -REACT_APP_CVIZ_API_WS_URL='ws://localhost:8089' + +REACT_APP_CVIZ_API_SERVER_URL='http://${DOCKER_HOST_IP}:8089' +REACT_APP_CVIZ_API_WS_URL='ws://${DOCKER_HOST_IP}:8089' # initial mapbox view REACT_APP_MAPBOX_INIT_LATITUDE=39.7392 diff --git a/webapp/Dockerfile b/webapp/Dockerfile index 5c2054e98..675351677 100644 --- a/webapp/Dockerfile +++ b/webapp/Dockerfile @@ -31,9 +31,6 @@ ENV REACT_APP_KEYCLOAK_REALM=$KEYCLOAK_REALM ARG KEYCLOAK_CLIENT_ID ENV REACT_APP_KEYCLOAK_CLIENT_ID=$KEYCLOAK_CLIENT_ID -ARG COUNT_MESSAGE_TYPES -ENV REACT_APP_COUNT_MESSAGE_TYPES=$COUNT_MESSAGE_TYPES - ARG VIEWER_MESSAGE_TYPES ENV REACT_APP_VIEWER_MESSAGE_TYPES=$VIEWER_MESSAGE_TYPES diff --git a/webapp/README.md b/webapp/README.md index d36779d43..b1c699e91 100644 --- a/webapp/README.md +++ b/webapp/README.md @@ -2,12 +2,13 @@ This is a web application that is made with React JS that is a front-end for interfacing with CDOT RSUs. The code for the application is being hosted in a public repository to allow other CV projects to utilize. Automated deployment of the application is not included within the project. -## Required Tools For Running The CV Manager React Webapp +## Requirements - Mapbox Access Token - Create account at https://www.mapbox.com/ - An access token will be provided on the account page once the account has been created - - Put the access key in the "sample.env.local" file for REACT_APP_MAPBOX_TOKEN and rename the file ".env.development.local" + - For more instructions, see the main README [Creating a Mapbox Token](../README.md#creating-a-mapbox-token) + - Set the access key in the "sample.env.local" file as the REACT_APP_MAPBOX_TOKEN - npm - Download instructions: https://docs.npmjs.com/downloading-and-installing-node-js-and-npm - Nodejs @@ -16,15 +17,18 @@ This is a web application that is made with React JS that is a front-end for int - Repository: https://github.com/usdot-jpo-ode/jpo-conflictmonitor - Follow the instructions in the README to build/deploy the tool -## Building The Application +## Running The Application -1. Setup environment variables - - The project includes a sample environment variable file in [sample.env.local](sample.env.local) for running the application locally. - - Use the dev, test and prod environment var files for deploying in different environments. The commented out environment variables are required. -2. Build the application - - Building for local: `npm run build` - - Building for specific environment: `npm run build:dev` - - Build for all environments: `npm run build:all` +1. Copy the `sample.env.local` file to a new file named `.env.local`. Make sure this new file is located in this directory (webapp), not root +2. Edit the `.env.local` file to set the required environment variables. At a minimum, you will need to set the following variables: + - `DOCKER_HOST_IP`: The IP address of your Docker host. This can be found through linux/wsl through the command "ifconfig", or "localhost" if using Docker Desktop on Windows or Linux (not mac). + - `REACT_APP_MAPBOX_TOKEN`: Any valid mapbox token. Please see the main README [Creating a Mapbox Token](../README.md#creating-a-mapbox-token) for instructions on how to create and account/generate a new token +3. Build the application + +```sh +npm install +npm start +``` ## Theming diff --git a/webapp/sample.env.local b/webapp/sample.env.local index 920aff320..067c68bda 100644 --- a/webapp/sample.env.local +++ b/webapp/sample.env.local @@ -1,29 +1,41 @@ # used to determine where to install build BUILD_PATH='./build' -# mapbox access token -REACT_APP_MAPBOX_TOKEN="" -REACT_APP_CVIZ_MAPBOX_TOKEN="" -REACT_APP_CVIZ_MAPBOX_STYLE_URL="" +############## REQUIRED ############## +DOCKER_HOST_IP= +REACT_APP_MAPBOX_TOKEN= # used to swap endpoints for various environments or leave blank REACT_APP_ENV="dev" # base url for api -REACT_APP_GATEWAY_BASE_URL="http://cvmanager.local.com:8081" +REACT_APP_GATEWAY_BASE_URL="http://${DOCKER_HOST_IP}:8081" # base url or IP for keycloak -REACT_APP_KEYCLOAK_URL="http://cvmanager.auth.com:8084/" +REACT_APP_KEYCLOAK_URL="http://${DOCKER_HOST_IP}:8084/" REACT_APP_KEYCLOAK_REALM="cvmanager" +REACT_APP_KEYCLOAK_CLIENT_ID=cvmanager-gui -REACT_APP_COUNT_MESSAGE_TYPES='BSM,SSM,SPAT,SRM,MAP,PSM' -REACT_APP_VIEWER_MESSAGE_TYPES='BSM,PSM' -DOT_NAME="CDOT" +REACT_APP_COUNT_MESSAGE_TYPES='BSM,SSM,SPAT,SRM,MAP' +REACT_APP_DOT_NAME="CDOT" -REACT_APP_CVIZ_API_SERVER_URL='http://localhost:8085' -REACT_APP_CVIZ_API_WS_URL='ws://localhost:8085' +REACT_APP_CVIZ_API_SERVER_URL='http://${DOCKER_HOST_IP}:8089' +REACT_APP_CVIZ_API_WS_URL='ws://${DOCKER_HOST_IP}:8089' # initial mapbox view -REACT_APP_MAPBOX_INIT_LATITUDE="39.7392" -REACT_APP_MAPBOX_INIT_LONGITUDE="-104.9903" -REACT_APP_MAPBOX_INIT_ZOOM="10" \ No newline at end of file +REACT_APP_MAPBOX_INIT_LATITUDE=39.7392 +REACT_APP_MAPBOX_INIT_LONGITUDE=-104.9903 +REACT_APP_MAPBOX_INIT_ZOOM=10 + +# 'true' to enable a feature and 'false' to disable it: +REACT_APP_ENABLE_RSU_FEATURES='true' +REACT_APP_ENABLE_INTERSECTION_FEATURES='true' +REACT_APP_ENABLE_WZDX_FEATURES='true' +REACT_APP_ENABLE_HAAS_FEATURES='true' + +# Webapp themes: dark +# base theme is used by default, dark theme is used if browser is set to dark mode +REACT_APP_WEBAPP_THEME_LIGHT="dark" +# if not set, defaults to 'light' +REACT_APP_WEBAPP_THEME_DARK="dark" +# if not set, defaults to 'dark' \ No newline at end of file diff --git a/webapp/src/App.test.tsx b/webapp/src/App.test.tsx index 232537c4f..c084dcccf 100644 --- a/webapp/src/App.test.tsx +++ b/webapp/src/App.test.tsx @@ -10,7 +10,6 @@ import { replaceChaoticIds } from './utils/test-utils' jest.mock('./EnvironmentVars', () => ({ WEBAPP_THEME_LIGHT: 'light', WEBAPP_THEME_DARK: 'dark', - getMessageTypes: jest.fn(() => ['BSM']), getMapboxInitViewState: jest.fn(() => ({ latitude: 39.7392, longitude: -104.9903, diff --git a/webapp/src/EnvironmentVars.test.tsx b/webapp/src/EnvironmentVars.test.tsx index 079897594..7618175f3 100644 --- a/webapp/src/EnvironmentVars.test.tsx +++ b/webapp/src/EnvironmentVars.test.tsx @@ -1,11 +1,5 @@ import EnvironmentVars from './EnvironmentVars' -it('returns message types', () => { - process.env.REACT_APP_COUNT_MESSAGE_TYPES = 'type1, type2, type3' - const expectedMessageTypes = ['type1', 'type2', 'type3'] - expect(EnvironmentVars.getMessageTypes()).toEqual(expectedMessageTypes) -}) - it('returns mapbox initial view state', () => { process.env.REACT_APP_MAPBOX_INIT_LATITUDE = '12.34' process.env.REACT_APP_MAPBOX_INIT_LONGITUDE = '56.78' diff --git a/webapp/src/EnvironmentVars.tsx b/webapp/src/EnvironmentVars.tsx index 0a6373f3a..a97aed098 100644 --- a/webapp/src/EnvironmentVars.tsx +++ b/webapp/src/EnvironmentVars.tsx @@ -3,15 +3,6 @@ class EnvironmentVars { return process.env.REACT_APP_GATEWAY_BASE_URL?.replace(/\/$/, '') // remove trailing slash } - static getMessageTypes() { - const COUNT_MESSAGE_TYPES = process.env.REACT_APP_COUNT_MESSAGE_TYPES - if (!COUNT_MESSAGE_TYPES) { - return [] - } - const messageTypes = COUNT_MESSAGE_TYPES.split(',').map((item) => item.trim()) - return messageTypes - } - static getMessageViewerTypes() { const VIEWER_MESSAGE_TYPES = process.env.REACT_APP_VIEWER_MESSAGE_TYPES if (!VIEWER_MESSAGE_TYPES) { diff --git a/webapp/src/components/HaasAlertVisualization.tsx b/webapp/src/components/HaasAlertVisualization.tsx index 39a0afb12..7f65723c2 100644 --- a/webapp/src/components/HaasAlertVisualization.tsx +++ b/webapp/src/components/HaasAlertVisualization.tsx @@ -82,9 +82,9 @@ export const HaasAlertVisualization: React.FC = ({ position: 'absolute', right: '10px', top: - menuSelection.includes('V2x Message Viewer') && menuSelection.includes('Configure RSUs') + menuSelection.includes('V2X Message Viewer') && menuSelection.includes('Configure RSUs') ? '540px' - : menuSelection.includes('V2x Message Viewer') + : menuSelection.includes('V2X Message Viewer') ? '380px' : menuSelection.includes('Configure RSUs') ? '180px' diff --git a/webapp/src/components/Help.test.tsx b/webapp/src/components/Help.test.tsx index 1f67244e2..cf721a6eb 100644 --- a/webapp/src/components/Help.test.tsx +++ b/webapp/src/components/Help.test.tsx @@ -2,9 +2,23 @@ import React from 'react' import { render } from '@testing-library/react' import Help from './Help' import { replaceChaoticIds } from '../utils/test-utils' +import {MemoryRouter} from "react-router-dom"; it('should take a snapshot', () => { - const { container } = render() + jest.mock('../EnvironmentVars', () => ({ + EnvironmentVars: { + ENABLE_RSU_FEATURES: true, + ENABLE_WZDX_FEATURES: true, + ENABLE_MOOVE_AI_FEATURES: true, + ENABLE_HAAS_FEATURES: true + } + })) + + const { container } = render( + + + + ) expect(replaceChaoticIds(container)).toMatchSnapshot() }) diff --git a/webapp/src/components/Help.tsx b/webapp/src/components/Help.tsx index c87d9d1ed..705bb1bac 100644 --- a/webapp/src/components/Help.tsx +++ b/webapp/src/components/Help.tsx @@ -1,69 +1,286 @@ import React from 'react' import '../components/css/Help.css' -import popup from '../icons/rsu_popup.PNG' -import status from '../icons/rsu_status.PNG' -import table from '../icons/rsu_count.PNG' -import menu from '../icons/rsu_menu.PNG' -import heatmap from '../icons/rsu_heatmap.PNG' -import configure from '../icons/rsu_configure.PNG' +import popup from '../icons/help/rsu_popup_and_config_menu.png' +import organizationSelection from '../icons/help/organization_selection.png' +import mapOverview from '../icons/help/map_overview.png' +import statusMenu from '../icons/help/rsu_status_menu.png' +import countMenu from '../icons/help/rsu_count_menu.png' +import table from '../icons/help/rsu_count.png' import EnvironmentVars from '../EnvironmentVars' import ContactSupportMenu from './ContactSupportMenu' import { BorderedImage } from '../styles/components/BorderedImage' -import { Stack, Container, useTheme } from '@mui/material' +import { + Stack, + Container, + useTheme, + TableContainer, + Table, + TableHead, + TableRow, + TableCell, + Paper, + TableBody, +} from '@mui/material' +import { Link } from 'react-router-dom' +import CheckCircleIcon from '@mui/icons-material/CheckCircle' +import CancelIcon from '@mui/icons-material/Cancel' const Help = () => { const theme = useTheme() + + const permissions = [ + { action: 'View Map Data (including RSUs)', user: true, operator: true, admin: true }, + { action: 'Register, Manage, and Configure RSUs', user: false, operator: true, admin: true }, + { action: 'Manage Intersection Settings and Notifications', user: false, operator: true, admin: true }, + { action: 'Manage Users', user: false, operator: false, admin: true }, + { action: 'Add and Remove Resources Within Organizations', user: false, operator: false, admin: true }, + ] + + const permissionsTable = ( + + + + + + User Roles and Permissions (By Organization) + + + + + {/* Empty cell for action column */} + + + User + + + Operator + + + Admin + + + + + {permissions.map((row, index) => ( + + {row.action} + + {row.user ? ( + + ) : ( + + )} + + + {row.operator ? ( + + ) : ( + + )} + + + {row.admin ? ( + + ) : ( + + )} + + + ))} + +
+ + ) + return ( -

Welcome to the {EnvironmentVars.DOT_NAME} CV Manager Website

+

{`Welcome to the ${EnvironmentVars.DOT_NAME} CV Manager`}

+

+ This application helps organizations manage and monitor deployed RSUs, monitor and detect issues with + connected intersections, and view numerous data types all in one application. +

+ +

Organizations and Profiles

+

+ This application uses organizations to manage permissions. All devices and users within the CV Manager are + associated with one or more organizations. When using the CV Manager, you can only view one organization at a + time. At the top right corner, you will see the User Profile menu. This menu allows the user to change + organizations, if they are a member of multiple, as well as logging out of the application. +

+ + + +

Permissions

+

- This application shows the physical location and message counts for each RSU installed by the Colorado - Department of Transportation at various road sites throughout Colorado. + There are 3 roles within the CV Manager: User, Operator, and Admin. Some users can also be granted Super User + permissions, which enables access to all resources within an organization and increases their ability to move + resources between organizations.

+ + {permissionsTable} + +

Enabled Features

- The map on this website will represent the location of each RSU with a red, green, or yellow dot (shown - below). A green dot represents an RSU that is online. A yellow dot represents an RSU that is offline but was - recently active. A red dot represents an RSU that is not currently online. + The CV Manager can have different feature sets enabled or disabled. In this application, the + following features are currently enabled or disabled as shown below:

- +
    +
  • {`RSU Monitoring and Configuration: ${EnvironmentVars.ENABLE_RSU_FEATURES ? 'ENABLED' : 'DISABLED'}`}
    • +
  • {`Intersection Map/Dashboard: ${EnvironmentVars.ENABLE_INTERSECTION_FEATURES ? 'ENABLED' : 'DISABLED'}`}
    • +
  • {`WZDx Viewer: ${EnvironmentVars.ENABLE_WZDX_FEATURES ? 'ENABLED' : 'DISABLED'}`}
    • +
  • {`Moove AI Viewer: ${EnvironmentVars.ENABLE_MOOVE_AI_FEATURES ? 'ENABLED' : 'DISABLED'}`}
    • +
  • {`HAAS Alert Viewer: ${EnvironmentVars.ENABLE_HAAS_FEATURES ? 'ENABLED' : 'DISABLED'}`}
    • +
+ +

+ Map Dashboard +

+

- Clicking on the RSU will give the following information: IP address, online status, time last online, milepost - number, serial number, and number of message counts (shown below). + The map dashboard is composed of a Mapbox map (background), the + Map Layer Menu (red), and + RSU Status and Message Counts Menu (green).

- + + + +

Map Layers

+

- The menu on the right side of the webpage will allow users to select the time range and message type to be - displayed for the RSUs. The menu will toggle between being hidden and visible each time the red X is clicked. - The time frame can be changed by adjusting the date and time values in the date-time pickers. The top - date-time picker represents the start time while the bottom represents the end time. Below the date-time - pickers is a dropdown menu where the RSU message type can be selected. Each time a value is updated in either - the date-time pickers or message selection dropdown the webpage will update accordingly. These elements are - shown in the image below: + The menu on the left contains three sections: Map Layers, RSU Filters, and + RSU Configuration. The Map Layers section allows users to visualize various data types. + Available layers include:

- + +
    +
  • + RSU Viewer: Displays RSU locations and status. Selecting one opens a popup and side panel + for configuration. Colors: +
      +
    • Green = online and reporting
      • +
    • Yellow = recently offline
      • +
    • Red = offline for extended time
      • +
    +
    • +
  • + Heatmap: Displays a message-count heatmap +
    • +
  • + V2X Message Viewer: Query messages for a map region +
    • +
  • + WZDx Viewer: Displays work zone events +
    • +
  • + Intersections: Shows connected intersections and IDs +
    • +
  • + Moove AI Viewer: Query harsh braking events +
    • +
  • + HAAS Alert Viewer: Query alert incidents by time +
    • +
+

- The table included in the menu will show the number of message counts for each RSU in the given time frame - regardless of online status (shown below). The table can be sorted by RSU, road, or count by clicking on the - desired column header. Each time a column header is clicked, it will toggle between sorting the data in an - ascending/descending fashion. + The RSU Filters section allows filtering RSUs by vendor and status. The RSU Configuration section allows + Operators/Admins to select RSUs on the map and configure them in bulk.

- + +

Configuring RSUs (Requires Operator or Admin)

+

- Selecting 'Heat Map' on the navigation bar will show a heat map representing RSU message counts for the time - range and message type selected in the menu (shown below). Any update to time range or message type in the - menu will carry over between the heat map and RSU map. + Selecting an RSU opens a popup and configuration menu. It displays IP address, online status, last report + time, milepost, serial number, and message count. Depending on access, users can retrieve configurations, + modify them, check/apply firmware updates, or reboot the RSU.

- + + + +
Message Forwarding Current Configuration
+

Message forwarding rules come in three types:

+ +
    +
  • TX (transmitted messages: TIM, MAP, SPAT, SSM)
    • +
  • RX (received messages: BSM, SRM, SDSM)
    • +
  • Generic (SNMP 4.1: applies to all messages)
    • +
+ +

Each rule includes:

+ +
    +
  • Message Type
    • +
  • Destination IP
    • +
  • Port
    • +
  • Start / End date
    • +
  • Security header enabled
    • +
  • Active state
    • +
  • Delete button
    • +
+ +
Message Forwarding Management
+

Create forwarding rules by entering:

+
    +
  • Destination IP
    • +
  • Message type
    • +
  • Security header
    • +
+ +
Firmware Management
+

Check for firmware updates and apply if available.

+ +
Reboot
+

Reboot the selected RSU.

+ +

RSU Status and Message Counts

+ +

+ On the right side are two menus: RSU Status Menu and + Message Count Menu. +

+ +

+ The RSU Status Menu lists all RSUs and their status. Users can print a full or error-only report. For each + device you will find: +

+ +
    +
  • Location
    • +
  • Online status (green / red / yellow)
    • +
  • SCMS certificate status
    • +
  • RSU IP address
    • +
+ + + +

+ The Message Count Menu filters RSU message counts by time range and message type. Changing any filter updates + the map and table automatically. +

+ + + +

A table below shows the number of messages from each RSU, sortable by RSU name, road, or message count.

+ + +

- Selecting 'Configure' on the navigation bar will allow users to perform certain actions to the RSU based on - their role. An RSU must first be selected on the RSU Map before any options become available. Users will be - able to perform one or more of the following: pull current message forwarding configuration, add/delete - message forwarding configurations, and perform an RSU reboot. + The RSU Configuration section allows applying configuration changes to multiple RSUs based on a selected + geographic region.

- - + ) } diff --git a/webapp/src/components/VerticalTabs.tsx b/webapp/src/components/VerticalTabs.tsx index cefd61527..ecee8c94b 100644 --- a/webapp/src/components/VerticalTabs.tsx +++ b/webapp/src/components/VerticalTabs.tsx @@ -1,11 +1,6 @@ import React, { useEffect, useState } from 'react' -import { useDispatch } from 'react-redux' -import { updateTableData as updateRsuTableData } from '../features/adminRsuTab/adminRsuTabSlice' -import { getAvailableUsers } from '../features/adminUserTab/adminUserTabSlice' import '../features/adminRsuTab/Admin.css' -import { AnyAction, ThunkDispatch } from '@reduxjs/toolkit' -import { RootState } from '../store' import { alpha, Box, Tab, Tabs, useTheme } from '@mui/material' import { Link, Navigate, Route, Routes, useLocation } from 'react-router-dom' import { evaluateFeatureFlags } from '../feature-flags' @@ -59,7 +54,6 @@ interface VerticalTabProps { function VerticalTabs(props: VerticalTabProps) { const { notFoundRoute, defaultTabIndex, tabs } = props - const dispatch: ThunkDispatch = useDispatch() const theme = useTheme() const location = useLocation() const filteredTabs = tabs.filter((tab) => evaluateFeatureFlags(tab.tag)) @@ -100,11 +94,6 @@ function VerticalTabs(props: VerticalTabProps) { setValue(newValue) } - useEffect(() => { - dispatch(updateRsuTableData()) - dispatch(getAvailableUsers()) - }, [dispatch]) - return ( -`; +`; \ No newline at end of file diff --git a/webapp/src/components/__snapshots__/Help.test.tsx.snap b/webapp/src/components/__snapshots__/Help.test.tsx.snap index 6e92642aa..78cb0f2bf 100644 --- a/webapp/src/components/__snapshots__/Help.test.tsx.snap +++ b/webapp/src/components/__snapshots__/Help.test.tsx.snap @@ -10,60 +10,626 @@ exports[`should take a snapshot 1`] = ` class="MuiStack-root css-mocked-MuiStack-root" >

- Welcome to the - CV Manager Website + Welcome to the undefined CV Manager

- This application shows the physical location and message counts for each RSU installed by the Colorado Department of Transportation at various road sites throughout Colorado. + This application helps organizations manage and monitor deployed RSUs, monitor and detect issues with connected intersections, and view numerous data types all in one application.

+

+ Organizations and Profiles +

- The map on this website will represent the location of each RSU with a red, green, or yellow dot (shown below). A green dot represents an RSU that is online. A yellow dot represents an RSU that is offline but was recently active. A red dot represents an RSU that is not currently online. + This application uses organizations to manage permissions. All devices and users within the CV Manager are associated with one or more organizations. When using the CV Manager, you can only view one organization at a time. At the top right corner, you will see the User Profile menu. This menu allows the user to change organizations, if they are a member of multiple, as well as logging out of the application.

RSU Statuses Shown on Map +

+ Permissions +

- Clicking on the RSU will give the following information: IP address, online status, time last online, milepost number, serial number, and number of message counts (shown below). + There are 3 roles within the CV Manager: User, Operator, and Admin. Some users can also be granted Super User permissions, which enables access to all resources within an organization and increases their ability to move resources between organizations. +

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ User Roles and Permissions (By Organization) +
+ + User + + Operator + + Admin +
+ View Map Data (including RSUs) + + + + + + +
+ Register, Manage, and Configure RSUs + + + + + + +
+ Manage Intersection Settings and Notifications + + + + + + +
+ Manage Users + + + + + + +
+ Add and Remove Resources Within Organizations + + + + + + +
+
+

+ Enabled Features +

+

+ The CV Manager can have different feature sets enabled or disabled. In this application, the following features are currently enabled or disabled as shown below: +

+
    +
  • + RSU Monitoring and Configuration: ENABLED +
    • +
  • + Intersection Map/Dashboard: ENABLED +
    • +
  • + WZDx Viewer: ENABLED +
    • +
  • + Moove AI Viewer: ENABLED +
    • +
  • + HAAS Alert Viewer: ENABLED +
    • +
+

+ + Map Dashboard + +

+

+ The map dashboard is composed of a Mapbox map (background), the + + Map Layer Menu + + (red), and + + RSU Status and Message Counts Menu + + (green).

RSU Popup with Data +

+ Map Layers +

+

+ The menu on the left contains three sections: + + Map Layers + + , + + RSU Filters + + , and + + RSU Configuration + + . The Map Layers section allows users to visualize various data types. Available layers include: +

+
    +
  • + + RSU Viewer: + + Displays RSU locations and status. Selecting one opens a popup and side panel for configuration. Colors: +
      +
    • + Green = online and reporting +
      • +
    • + Yellow = recently offline +
      • +
    • + Red = offline for extended time +
      • +
    +
    • +
  • + + Heatmap: + + Displays a message-count heatmap +
    • +
  • + + V2X Message Viewer: + + Query messages for a map region +
    • +
  • + + WZDx Viewer: + + Displays work zone events +
    • +
  • + + Intersections: + + Shows connected intersections and IDs +
    • +
  • + + Moove AI Viewer: + + Query harsh braking events +
    • +
  • + + HAAS Alert Viewer: + + Query alert incidents by time +
    • +
+

+ The RSU Filters section allows filtering RSUs by vendor and status. The RSU Configuration section allows Operators/Admins to select RSUs on the map and configure them in bulk. +

+

+ Configuring RSUs (Requires Operator or Admin) +

- The menu on the right side of the webpage will allow users to select the time range and message type to be displayed for the RSUs. The menu will toggle between being hidden and visible each time the red X is clicked. The time frame can be changed by adjusting the date and time values in the date-time pickers. The top date-time picker represents the start time while the bottom represents the end time. Below the date-time pickers is a dropdown menu where the RSU message type can be selected. Each time a value is updated in either the date-time pickers or message selection dropdown the webpage will update accordingly. These elements are shown in the image below: + Selecting an RSU opens a popup and configuration menu. It displays IP address, online status, last report time, milepost, serial number, and message count. Depending on access, users can retrieve configurations, modify them, check/apply firmware updates, or reboot the RSU.

RSU Menu +
+ Message Forwarding Current Configuration +

- The table included in the menu will show the number of message counts for each RSU in the given time frame regardless of online status (shown below). The table can be sorted by RSU, road, or count by clicking on the desired column header. Each time a column header is clicked, it will toggle between sorting the data in an ascending/descending fashion. + Message forwarding rules come in three types:

+
    +
  • + TX (transmitted messages: TIM, MAP, SPAT, SSM) +
    • +
  • + RX (received messages: BSM, SRM, SDSM) +
    • +
  • + Generic (SNMP 4.1: applies to all messages) +
    • +
+

+ Each rule includes: +

+
    +
  • + Message Type +
    • +
  • + Destination IP +
    • +
  • + Port +
    • +
  • + Start / End date +
    • +
  • + Security header enabled +
    • +
  • + Active state +
    • +
  • + Delete button +
    • +
+
+ Message Forwarding Management +
+

+ Create forwarding rules by entering: +

+
    +
  • + Destination IP +
    • +
  • + Message type +
    • +
  • + Security header +
    • +
+
+ Firmware Management +
+

+ Check for firmware updates and apply if available. +

+
+ Reboot +
+

+ Reboot the selected RSU. +

+

+ RSU Status and Message Counts +

+

+ On the right side are two menus: + + RSU Status Menu + + and + + Message Count Menu + + . +

+

+ The RSU Status Menu lists all RSUs and their status. Users can print a full or error-only report. For each device you will find: +

+
    +
  • + Location +
    • +
  • + Online status (green / red / yellow) +
    • +
  • + SCMS certificate status +
    • +
  • + RSU IP address +
    • +
RSU Message Count Table

- Selecting 'Heat Map' on the navigation bar will show a heat map representing RSU message counts for the time range and message type selected in the menu (shown below). Any update to time range or message type in the menu will carry over between the heat map and RSU map. + The Message Count Menu filters RSU message counts by time range and message type. Changing any filter updates the map and table automatically.

CV Manager Heat Map

- Selecting 'Configure' on the navigation bar will allow users to perform certain actions to the RSU based on their role. An RSU must first be selected on the RSU Map before any options become available. Users will be able to perform one or more of the following: pull current message forwarding configuration, add/delete message forwarding configurations, and perform an RSU reboot. + A table below shows the number of messages from each RSU, sortable by RSU name, road, or message count.

CV Manager Configuration Page +

+ The RSU Configuration section allows applying configuration changes to multiple RSUs based on a selected geographic region. +

{ const navigate = useNavigate() const theme = useTheme() + const organization = useSelector(selectOrganizationName) + useEffect(() =>{ + dispatch(updateTableData()) + }, [organization, dispatch]) + const tableData = useSelector(selectTableData) const columns = useSelector(selectColumns) @@ -94,7 +100,7 @@ const AdminIntersectionTab = () => { itemType: 'outlined', }, onClick: () => { - updateTableData() + dispatch(updateTableData()) }, }, { diff --git a/webapp/src/features/adminIntersectionTab/adminIntersectionTabSlice.tsx b/webapp/src/features/adminIntersectionTab/adminIntersectionTabSlice.tsx index 332cb5f3e..e8da269c3 100644 --- a/webapp/src/features/adminIntersectionTab/adminIntersectionTabSlice.tsx +++ b/webapp/src/features/adminIntersectionTab/adminIntersectionTabSlice.tsx @@ -1,5 +1,5 @@ import { createAsyncThunk, createSlice } from '@reduxjs/toolkit' -import { selectToken } from '../../generalSlices/userSlice' +import {selectOrganizationName, selectToken} from '../../generalSlices/userSlice' import EnvironmentVars from '../../EnvironmentVars' import apiHelper from '../../apis/api-helper' import { RootState } from '../../store' @@ -29,12 +29,12 @@ export const updateTableData = createAsyncThunk( async (_, { getState }) => { const currentState = getState() as RootState const token = selectToken(currentState) - + const organization = selectOrganizationName(currentState) const data = await apiHelper._getDataWithCodes({ url: EnvironmentVars.adminIntersection, token, query_params: { intersection_id: 'all' }, - additional_headers: { 'Content-Type': 'application/json' }, + additional_headers: { 'Content-Type': 'application/json', Organization: organization }, tag: 'intersection', }) @@ -165,4 +165,4 @@ export const selectColumns = (state: RootState) => state.adminIntersectionTab.va export const selectEditIntersectionRowData = (state: RootState) => state.adminIntersectionTab.value.editIntersectionRowData -export default adminIntersectionTabSlice.reducer +export default adminIntersectionTabSlice.reducer \ No newline at end of file diff --git a/webapp/src/features/adminRsuTab/AdminRsuTab.tsx b/webapp/src/features/adminRsuTab/AdminRsuTab.tsx index 731ccd082..a422351bc 100644 --- a/webapp/src/features/adminRsuTab/AdminRsuTab.tsx +++ b/webapp/src/features/adminRsuTab/AdminRsuTab.tsx @@ -1,4 +1,4 @@ -import React, { useState } from 'react' +import React, { useState, useEffect } from 'react' import AdminAddRsu from '../adminAddRsu/AdminAddRsu' import AdminEditRsu, { AdminEditRsuFormType } from '../adminEditRsu/AdminEditRsu' import AdminTable from '../../components/AdminTable' @@ -14,6 +14,7 @@ import { deleteRsu, setEditRsuRowData, } from './adminRsuTabSlice' +import { selectOrganizationName } from '../../generalSlices/userSlice' import { clear, getRsuInfo } from '../adminEditRsu/adminEditRsuSlice' import { useSelector, useDispatch } from 'react-redux' @@ -31,6 +32,10 @@ const AdminRsuTab = () => { const dispatch: ThunkDispatch = useDispatch() const navigate = useNavigate() const theme = useTheme() + const organization = useSelector(selectOrganizationName) + useEffect(() =>{ + dispatch(updateTableData()) + }, [organization, dispatch]) const tableData = useSelector(selectTableData) const [columns] = useState([ @@ -100,7 +105,7 @@ const AdminRsuTab = () => { }, position: 'toolbar', onClick: () => { - updateTableData() + dispatch(updateTableData()) }, }, { diff --git a/webapp/src/features/adminRsuTab/adminRsuTabSlice.test.ts b/webapp/src/features/adminRsuTab/adminRsuTabSlice.test.ts index d1d2ddcc3..8c756f03e 100644 --- a/webapp/src/features/adminRsuTab/adminRsuTabSlice.test.ts +++ b/webapp/src/features/adminRsuTab/adminRsuTabSlice.test.ts @@ -80,7 +80,7 @@ describe('async thunks', () => { additional_headers: { 'Content-Type': 'application/json' }, tag: 'rsu', }) - expect(dispatch).toHaveBeenCalledTimes(1 + 2) + expect(dispatch).toHaveBeenCalledTimes(0 + 2) dispatch = jest.fn() apiHelper._getDataWithCodes = jest.fn().mockReturnValue({ status: 500, message: 'message' }) @@ -92,7 +92,7 @@ describe('async thunks', () => { additional_headers: { 'Content-Type': 'application/json' }, tag: 'rsu', }) - expect(dispatch).toHaveBeenCalledTimes(1 + 2) + expect(dispatch).toHaveBeenCalledTimes(0 + 2) }) it('Updates the state correctly pending', async () => { diff --git a/webapp/src/features/adminRsuTab/adminRsuTabSlice.tsx b/webapp/src/features/adminRsuTab/adminRsuTabSlice.tsx index f22760934..40b9a8a63 100644 --- a/webapp/src/features/adminRsuTab/adminRsuTabSlice.tsx +++ b/webapp/src/features/adminRsuTab/adminRsuTabSlice.tsx @@ -1,10 +1,9 @@ -import { createAsyncThunk, createSlice } from '@reduxjs/toolkit' -import { selectToken } from '../../generalSlices/userSlice' +import {createAsyncThunk, createSlice} from '@reduxjs/toolkit' +import {selectOrganizationName, selectToken} from '../../generalSlices/userSlice' import EnvironmentVars from '../../EnvironmentVars' import apiHelper from '../../apis/api-helper' -import { getRsuInfoOnly } from '../../generalSlices/rsuSlice' -import { RootState } from '../../store' -import { AdminEditRsuFormType } from '../adminEditRsu/AdminEditRsu' +import {RootState} from '../../store' +import {AdminEditRsuFormType} from '../adminEditRsu/AdminEditRsu' const initialState = { tableData: [] as AdminEditRsuFormType[], @@ -21,17 +20,16 @@ const initialState = { export const updateTableData = createAsyncThunk( 'adminRsuTab/updateTableData', - async (_, { getState, dispatch }) => { + async (_, { getState }) => { const currentState = getState() as RootState const token = selectToken(currentState) - - dispatch(getRsuInfoOnly()) + const organization = selectOrganizationName(currentState) const data = await apiHelper._getDataWithCodes({ url: EnvironmentVars.adminRsu, token, query_params: { rsu_ip: 'all' }, - additional_headers: { 'Content-Type': 'application/json' }, + additional_headers: { 'Content-Type': 'application/json', Organization: organization }, tag: 'rsu', }) diff --git a/webapp/src/features/adminUserTab/AdminUserTab.tsx b/webapp/src/features/adminUserTab/AdminUserTab.tsx index 782a70445..6209b338e 100644 --- a/webapp/src/features/adminUserTab/AdminUserTab.tsx +++ b/webapp/src/features/adminUserTab/AdminUserTab.tsx @@ -15,6 +15,7 @@ import { setActiveDiv, setEditUserRowData, } from './adminUserTabSlice' +import { selectOrganizationName } from '../../generalSlices/userSlice' import { clear, getUserData } from './../adminEditUser/adminEditUserSlice' import { useSelector, useDispatch } from 'react-redux' @@ -33,6 +34,10 @@ const AdminUserTab = () => { const navigate = useNavigate() const location = useLocation() const theme = useTheme() + const organization = useSelector(selectOrganizationName) + useEffect(() =>{ + dispatch(getAvailableUsers()) + }, [organization, dispatch]) const activeTab = location.pathname.split('/')[4] @@ -115,7 +120,7 @@ const AdminUserTab = () => { itemType: 'outlined', }, onClick: () => { - updateTableData() + dispatch(getAvailableUsers()) }, }, { @@ -142,10 +147,6 @@ const AdminUserTab = () => { }) } - const updateTableData = async () => { - dispatch(getAvailableUsers()) - } - useEffect(() => { dispatch(setActiveDiv('user_table')) }, [dispatch]) diff --git a/webapp/src/features/adminUserTab/adminUserTabSlice.tsx b/webapp/src/features/adminUserTab/adminUserTabSlice.tsx index 53b3a31b3..abbd702a2 100644 --- a/webapp/src/features/adminUserTab/adminUserTabSlice.tsx +++ b/webapp/src/features/adminUserTab/adminUserTabSlice.tsx @@ -1,5 +1,5 @@ import { createAsyncThunk, createSlice } from '@reduxjs/toolkit' -import { selectToken } from '../../generalSlices/userSlice' +import {selectOrganizationName, selectToken} from '../../generalSlices/userSlice' import EnvironmentVars from '../../EnvironmentVars' import apiHelper from '../../apis/api-helper' import { RootState } from '../../store' @@ -11,12 +11,12 @@ const initialState = { editUserRowData: {} as AdminUserWithId, } -export const getUserData = async (user_email: string, token: string) => { +export const getUserData = async (user_email: string, token: string, organization: string) => { return await apiHelper._getDataWithCodes({ url: EnvironmentVars.adminUser, token, query_params: { user_email }, - additional_headers: { 'Content-Type': 'application/json' }, + additional_headers: { 'Content-Type': 'application/json', Organization: organization }, }) } @@ -42,8 +42,9 @@ export const getAvailableUsers = createAsyncThunk( async (_, { getState }) => { const currentState = getState() as RootState const token = selectToken(currentState) + const organization = selectOrganizationName(currentState) - const data = await getUserData('all', token) + const data = await getUserData('all', token, organization) switch (data.status) { case 200: diff --git a/webapp/src/features/api/intersectionApiSlice.ts b/webapp/src/features/api/intersectionApiSlice.ts index 89a134aa1..6647f3868 100644 --- a/webapp/src/features/api/intersectionApiSlice.ts +++ b/webapp/src/features/api/intersectionApiSlice.ts @@ -7,7 +7,7 @@ import { selectToken } from '../../generalSlices/userSlice' import { selectSelectedIntersectionId } from '../../generalSlices/intersectionSlice' import { combineUrlPaths } from '../../apis/intersections/api-helper-cviz' -const getQueryString = (query_params: Record) => { +export const getQueryString = (query_params: Record) => { // filter out undefined values from query params const filteredQueryParams: Record = { ...query_params } Object.keys(filteredQueryParams).forEach((key) => query_params[key] === undefined && delete query_params[key]) diff --git a/webapp/src/features/api/rsuCountsApiSlice.ts b/webapp/src/features/api/rsuCountsApiSlice.ts new file mode 100644 index 000000000..9f545c3bf --- /dev/null +++ b/webapp/src/features/api/rsuCountsApiSlice.ts @@ -0,0 +1,47 @@ +// Need to use the React-specific entry point to import createApi +import { createApi, fetchBaseQuery } from '@reduxjs/toolkit/query/react' +import EnvironmentVars from '../../EnvironmentVars' +import { RootState } from '../../store' +import { selectToken } from '../../generalSlices/userSlice' +import { RsuCounts } from '../../models/RsuApi' +import { getQueryString } from './intersectionApiSlice' + +// Define a service using a base URL and expected endpoints +export const rsuCountsApiSlice = createApi({ + reducerPath: 'rsuCountsApi', + baseQuery: fetchBaseQuery({ + baseUrl: EnvironmentVars.cvmanagerBaseEndpoint, + prepareHeaders: (headers, { getState, endpoint }) => { + const currentState = getState() as RootState + const token = selectToken(currentState) + + // Specify endpoints that do not require a token or organization. These names must match the keys in the endpoints object below. + const endpointsWithoutToken = [] + + if (token && !endpointsWithoutToken.includes(endpoint)) { + headers.set('Authorization', `${token}`) + } + + return headers + }, + }), + endpoints: (builder) => ({ + getRsuCounts: builder.query({ + query: ({ organization, startDate, endDate }) => { + return { + url: `/rsucounts${getQueryString({ + start: startDate.toISOString(), + end: endDate.toISOString(), + })}`, + headers: { + Organization: organization, + }, + } + }, + }), + }), +}) + +// Export hooks for usage in functional components, which are +// auto-generated based on the defined endpoints +export const { useGetRsuCountsQuery, useLazyGetRsuCountsQuery, util: rsuCountsApiUtil } = rsuCountsApiSlice diff --git a/webapp/src/features/intersections/reports/report-list-filters.test.tsx b/webapp/src/features/intersections/reports/report-list-filters.test.tsx index 62ba3f942..daa84a120 100644 --- a/webapp/src/features/intersections/reports/report-list-filters.test.tsx +++ b/webapp/src/features/intersections/reports/report-list-filters.test.tsx @@ -16,12 +16,33 @@ jest.mock('@mui/x-date-pickers', () => { } }) -// Mock the dayjs library +// Mock the dayjs library with timezone support jest.mock('dayjs', () => { const actualDayjs = jest.requireActual('dayjs') - const mockDayjs = (date) => actualDayjs(date).tz('America/Denver') + const utc = require('dayjs/plugin/utc') + const timezone = require('dayjs/plugin/timezone') + + // Extend dayjs with required plugins + actualDayjs.extend(utc) + actualDayjs.extend(timezone) + + // Create mock function + const mockDayjs = (date?: any) => { + const instance = date ? actualDayjs(date) : actualDayjs() + return instance.tz('America/Denver') + } + + // Copy all static methods and properties from actual dayjs + Object.keys(actualDayjs).forEach((key) => { + if (!(key in mockDayjs)) { + mockDayjs[key] = actualDayjs[key] + } + }) + + // Ensure timezone method is available mockDayjs.extend = actualDayjs.extend mockDayjs.Ls = actualDayjs.Ls + return mockDayjs }) diff --git a/webapp/src/features/intersections/reports/report-request-edit-form.test.tsx b/webapp/src/features/intersections/reports/report-request-edit-form.test.tsx index bc3102fcb..4dae65cfe 100644 --- a/webapp/src/features/intersections/reports/report-request-edit-form.test.tsx +++ b/webapp/src/features/intersections/reports/report-request-edit-form.test.tsx @@ -16,12 +16,33 @@ jest.mock('@mui/x-date-pickers', () => { } }) -// Mock the dayjs library +// Mock the dayjs library with timezone support jest.mock('dayjs', () => { const actualDayjs = jest.requireActual('dayjs') - const mockDayjs = (date) => actualDayjs(date).tz('America/Denver') + const utc = require('dayjs/plugin/utc') + const timezone = require('dayjs/plugin/timezone') + + // Extend dayjs with required plugins + actualDayjs.extend(utc) + actualDayjs.extend(timezone) + + // Create mock function + const mockDayjs = (date?: any) => { + const instance = date ? actualDayjs(date) : actualDayjs() + return instance.tz('America/Denver') + } + + // Copy all static methods and properties from actual dayjs + Object.keys(actualDayjs).forEach((key) => { + if (!(key in mockDayjs)) { + mockDayjs[key] = actualDayjs[key] + } + }) + + // Ensure timezone method is available mockDayjs.extend = actualDayjs.extend mockDayjs.Ls = actualDayjs.Ls + return mockDayjs }) diff --git a/webapp/src/features/menu/ConfigureRSU.tsx b/webapp/src/features/menu/ConfigureRSU.tsx index 3739908d4..eb65a5e5b 100644 --- a/webapp/src/features/menu/ConfigureRSU.tsx +++ b/webapp/src/features/menu/ConfigureRSU.tsx @@ -19,6 +19,8 @@ import CloseIcon from '@mui/icons-material/Close' import { RoomOutlined } from '@mui/icons-material' import { headerTabHeight } from '../../styles' import { SideBarHeader } from '../../styles/components/SideBarHeader' +import { CustomTable } from '../intersections/map/custom-table' +import EnvironmentVars from '../../EnvironmentVars' const ConfigMenu = ({ children }) => { return {children} @@ -88,6 +90,23 @@ const ConfigureRSU = () => { color: theme.palette.text.secondary, }} > + + } aria-controls="panel1bh-content" id="panel1bh-header"> + Message Counts + + + { + return [type, count?.toString() ?? '0'] + })} + headers={['Msg Type', 'Count']} + /> + + { @@ -16,19 +17,52 @@ jest.mock('@mui/x-date-pickers', () => { } }) -// Mock the dayjs library +// Mock the dayjs library with timezone support jest.mock('dayjs', () => { const actualDayjs = jest.requireActual('dayjs') - const mockDayjs = (date) => actualDayjs(date).tz('America/Denver') + const utc = require('dayjs/plugin/utc') + const timezone = require('dayjs/plugin/timezone') + + // Extend dayjs with required plugins + actualDayjs.extend(utc) + actualDayjs.extend(timezone) + + // Create mock function + const mockDayjs = (date?: any) => { + const instance = date ? actualDayjs(date) : actualDayjs() + return instance.tz('America/Denver') + } + + // Copy all static methods and properties from actual dayjs + Object.keys(actualDayjs).forEach((key) => { + if (!(key in mockDayjs)) { + mockDayjs[key] = actualDayjs[key] + } + }) + + // Ensure timezone method is available mockDayjs.extend = actualDayjs.extend mockDayjs.Ls = actualDayjs.Ls + return mockDayjs }) it('should take a snapshot', () => { const { container } = render( - + diff --git a/webapp/src/features/menu/DisplayCounts.tsx b/webapp/src/features/menu/DisplayCounts.tsx index 458440e65..10f28bcf4 100644 --- a/webapp/src/features/menu/DisplayCounts.tsx +++ b/webapp/src/features/menu/DisplayCounts.tsx @@ -1,4 +1,4 @@ -import React from 'react' +import React, { useMemo } from 'react' import { useSelector, useDispatch } from 'react-redux' import dayjs from 'dayjs' import { LocalizationProvider } from '@mui/x-date-pickers/LocalizationProvider' @@ -6,22 +6,13 @@ import { AdapterDayjs } from '@mui/x-date-pickers/AdapterDayjs' import { DateTimePicker } from '@mui/x-date-pickers/DateTimePicker' import EnvironmentVars from '../../EnvironmentVars' -import BounceLoader from 'react-spinners/BounceLoader' import { - selectRequestOut, - selectMsgType, - selectCountList, - selectStartDate, - selectEndDate, - selectWarningMessage, - selectMessageLoading, - updateMessageType, -} from '../../generalSlices/rsuSlice' -import { - selectCurrentSort, - selectSortedCountList, - sortCountList, - changeDate, + selectCountsEndDate, + selectCountsMsgType, + selectCountsStartDate, + setCountsEndDate, + setCountsMsgType, + setCountsStartDate, toggleMapMenuSelection, } from './menuSlice' @@ -32,29 +23,48 @@ import { CountsListElement } from '../../models/Rsu' import { MessageType } from '../../models/MessageTypes' import { Box, FormControl, InputLabel, MenuItem, Paper, Select, Stack, Typography, useTheme } from '@mui/material' import { SideBarHeader } from '../../styles/components/SideBarHeader' - -const messageTypeOptions = EnvironmentVars.getMessageTypes().map((type) => { - return { value: type, label: type } -}) +import { useGetRsuCountsQuery } from '../api/rsuCountsApiSlice' +import { selectOrganizationName } from '../../generalSlices/userSlice' const DisplayCounts = () => { const dispatch: ThunkDispatch = useDispatch() const theme = useTheme() - const countsMsgType = useSelector(selectMsgType) - const startDate = useSelector(selectStartDate) - const endDate = useSelector(selectEndDate) - const requestOut = useSelector(selectRequestOut) - const warning = useSelector(selectWarningMessage) - const messageLoading = useSelector(selectMessageLoading) - const countList = useSelector(selectCountList) - const currentSort = useSelector(selectCurrentSort) - const sortedCountList = useSelector(selectSortedCountList) - - const dateChanged = (e: Date, type: 'start' | 'end') => { - if (!Number.isNaN(Date.parse(e.toString()))) { - dispatch(changeDate(e, type, requestOut)) + const organization = useSelector(selectOrganizationName) + const countsMsgType = useSelector(selectCountsMsgType) + const startDate = useSelector(selectCountsStartDate) + const endDate = useSelector(selectCountsEndDate) + + const [currentSort, setCurrentSort] = React.useState(null) + + const { data: rsuCounts } = useGetRsuCountsQuery({ organization, startDate, endDate }) + + const messageTypeOptions = useMemo(() => { + // parse message types from rsuCounts count keys + const typesSet = new Set() + Object.values(rsuCounts || {}).forEach((rsuCount) => { + Object.keys(rsuCount.messageTypeCounts || {}).forEach((msgType) => typesSet.add(msgType)) + }) + if (typesSet.size === 0) { + return [{ value: 'BSM', label: 'BSM' }] } - } + return Array.from(typesSet).map((type) => ({ value: type, label: type?.toUpperCase() })) + }, [rsuCounts]) + + const countList = useMemo(() => { + return Object.entries(rsuCounts ?? {}).map(([key, value]) => { + return { + key: key, + rsu: key, + road: value.road, + count: value.messageTypeCounts?.[countsMsgType] || 0, + } + }) + }, [rsuCounts, countsMsgType]) + + const dateRangeValid = useMemo(() => { + const ONE_WEEK_MILLISECONDS = 86400 * 1000 * 7 + return endDate.getTime() - startDate.getTime() > ONE_WEEK_MILLISECONDS + }, [startDate, endDate]) const getWarningMessage = (warning: boolean) => warning ? ( @@ -63,44 +73,52 @@ const DisplayCounts = () => { role="alert" sx={{ backgroundColor: theme.palette.error.main, display: 'flex', justifyContent: 'center' }} > - Warning: time ranges greater than 24 hours may have longer load times. + Warning: time ranges greater than 7 days may have longer load times. ) : null + const sortedCountList = useMemo(() => { + if (!currentSort) return countList + + const key = currentSort.replace('__desc', '') + const isDescending = currentSort.includes('__desc') + + return [...countList].sort((a, b) => { + const aVal = a[key] + const bVal = b[key] + + if (aVal < bVal) return isDescending ? 1 : -1 + if (aVal > bVal) return isDescending ? -1 : 1 + return 0 + }) + }, [currentSort, countList]) + const sortBy = (key: string) => { - dispatch(sortCountList(key, currentSort, countList)) + // Default to ascending. If re-pressed (already sorting by this key), switch to descending. + if (key === currentSort) { + setCurrentSort(key + '__desc') + } else { + setCurrentSort(key) + } } - const getTable = (messageLoading: boolean, sortedCountList: CountsListElement[]) => - messageLoading ? ( -
-
-
-
RSU
-
Road
-
Count
-
+ const getTable = (sortedCountList: CountsListElement[]) => ( +
+
+
sortBy('rsu')} style={{ borderBottom: `1px solid ${theme.palette.text.primary}` }}> + RSU
- - - -
- ) : ( -
-
-
sortBy('rsu')} style={{ borderBottom: `1px solid ${theme.palette.text.primary}` }}> - RSU -
-
sortBy('road')} style={{ borderBottom: `1px solid ${theme.palette.text.primary}` }}> - Road -
-
sortBy('count')} style={{ borderBottom: `1px solid ${theme.palette.text.primary}` }}> - Count -
+
sortBy('road')} style={{ borderBottom: `1px solid ${theme.palette.text.primary}` }}> + Road +
+
sortBy('count')} style={{ borderBottom: `1px solid ${theme.palette.text.primary}` }}> + Count
-
{formatRows(sortedCountList)}
- ) +
{formatRows(sortedCountList)}
+
+ ) + const formatRows = (rows: CountsListElement[]) => { if (rows.length === 0) { return ( @@ -133,8 +151,9 @@ const DisplayCounts = () => { value={dayjs(startDate)} maxDateTime={dayjs(endDate)} onChange={(e) => { - if (e === null) return - dateChanged(e.toDate(), 'start') + if (e && !Number.isNaN(Date.parse(e.toString()))) { + dispatch(setCountsStartDate(e.toDate())) + } }} /> @@ -148,8 +167,9 @@ const DisplayCounts = () => { minDateTime={dayjs(startDate)} maxDateTime={dayjs(endDate)} onChange={(e) => { - if (e === null) return - dateChanged(e.toDate(), 'end') + if (e && !Number.isNaN(Date.parse(e.toString()))) { + dispatch(setCountsEndDate(e.toDate())) + } }} /> @@ -161,7 +181,7 @@ const DisplayCounts = () => { label="Message Type" id="counts-msg-dropdown" value={countsMsgType} - onChange={(event) => dispatch(updateMessageType(event.target.value as MessageType))} + onChange={(event) => dispatch(setCountsMsgType(event.target.value as MessageType))} sx={{ textAlign: 'left', }} @@ -176,8 +196,8 @@ const DisplayCounts = () => { - {getWarningMessage(warning)} - {getTable(messageLoading, sortedCountList)} + {getWarningMessage(dateRangeValid)} + {getTable(sortedCountList)} ) diff --git a/webapp/src/features/menu/Menu.tsx b/webapp/src/features/menu/Menu.tsx index 76fa654f2..a63d01f85 100644 --- a/webapp/src/features/menu/Menu.tsx +++ b/webapp/src/features/menu/Menu.tsx @@ -1,10 +1,9 @@ import React from 'react' import './Menu.css' -import { useEffect } from 'react' import { useSelector, useDispatch } from 'react-redux' -import { selectCountList, selectSelectedRsu } from '../../generalSlices/rsuSlice' +import { selectSelectedRsu } from '../../generalSlices/rsuSlice' import { selectConfigList } from '../../generalSlices/configSlice' -import { selectDisplayCounts, setSortedCountList, selectDisplayRsuErrors } from './menuSlice' +import { selectDisplayCounts, selectDisplayRsuErrors } from './menuSlice' import { SecureStorageManager } from '../../managers' import DisplayCounts from './DisplayCounts' import DisplayRsuErrors from './DisplayRsuErrors' @@ -27,16 +26,11 @@ const menuStyle: React.CSSProperties = { const Menu = () => { const dispatch: ThunkDispatch = useDispatch() const theme = useTheme() - const countList = useSelector(selectCountList) const selectedRsu = useSelector(selectSelectedRsu) const selectedRsuList = useSelector(selectConfigList) const displayCounts = useSelector(selectDisplayCounts) const displayRsuErrors = useSelector(selectDisplayRsuErrors) - useEffect(() => { - dispatch(setSortedCountList(countList)) - }, [countList, dispatch]) - return (
{displayCounts === true && !selectedRsu && selectedRsuList?.length === 0 && ( diff --git a/webapp/src/features/menu/__snapshots__/DisplayCounts.test.tsx.snap b/webapp/src/features/menu/__snapshots__/DisplayCounts.test.tsx.snap index 27651eeea..7ecec3fb1 100644 --- a/webapp/src/features/menu/__snapshots__/DisplayCounts.test.tsx.snap +++ b/webapp/src/features/menu/__snapshots__/DisplayCounts.test.tsx.snap @@ -187,11 +187,7 @@ exports[`should take a snapshot 1`] = ` role="combobox" tabindex="0" > - - ​ - + BSM
{ + const actualDayjs = jest.requireActual('dayjs') + const mockNow = actualDayjs('2024-01-15T12:00:00.000Z') + + const dayjsMock = jest.fn((date?: any) => { + if (date) { + return actualDayjs(date) + } + return mockNow + }) + + // Copy all dayjs methods + Object.keys(actualDayjs).forEach((key) => { + dayjsMock[key] = actualDayjs[key] + }) + + return dayjsMock +}) describe('menu reducer', () => { it('should handle initial state', () => { - expect(reducer(undefined, { type: 'unknown' })).toEqual({ + const expected = { loading: false, value: { - currentSort: null, - sortedCountList: [], + countsMsgType: 'BSM', + countsStartDate: expect.any(Date), + countsEndDate: expect.any(Date), displayCounts: false, displayRsuErrors: false, menuSelection: [], }, - }) + } + + const actual = reducer(undefined, { type: 'unknown' }) + expect(actual).toEqual(expected) + + // Verify dates are set correctly (yesterday and today) + const startDate = new Date(actual.value.countsStartDate) + const endDate = new Date(actual.value.countsEndDate) + expect(endDate.getTime() - startDate.getTime()).toBe(24 * 60 * 60 * 1000) // 1 day difference }) }) describe('reducers', () => { const initialState: RootState['menu'] = { - loading: null, + loading: false, value: { - currentSort: null, - sortedCountList: null, + countsMsgType: 'BSM', + countsStartDate: new Date('2024-01-14T12:00:00.000Z'), + countsEndDate: new Date('2024-01-15T12:00:00.000Z'), displayCounts: false, displayRsuErrors: false, menuSelection: [], }, } - it('setCurrentSort reducer updates state correctly', async () => { - const currentSort = 'currentSort' - expect(reducer(initialState, setCurrentSort(currentSort))).toEqual({ + it('setCountsMsgType reducer updates state correctly', () => { + const newMsgType = 'SPaT' + expect(reducer(initialState, setCountsMsgType(newMsgType))).toEqual({ ...initialState, - value: { ...initialState.value, currentSort }, + value: { ...initialState.value, countsMsgType: newMsgType }, }) }) - it('setSortedCountList reducer updates state correctly', async () => { - const sortedCountList = 'sortedCountList' - expect(reducer(initialState, setSortedCountList(sortedCountList))).toEqual({ + it('setCountsMsgType handles different message types', () => { + const messageTypes = ['BSM', 'SPaT', 'MAP', 'SSM', 'SRM', 'TIM', 'PSM'] + + messageTypes.forEach((msgType) => { + const result = reducer(initialState, setCountsMsgType(msgType as any)) + expect(result.value.countsMsgType).toBe(msgType) + }) + }) + + it('setCountsStartDate reducer updates state correctly', () => { + const newStartDate = new Date('2024-01-10T12:00:00.000Z') + expect(reducer(initialState, setCountsStartDate(newStartDate))).toEqual({ ...initialState, - value: { ...initialState.value, sortedCountList }, + value: { ...initialState.value, countsStartDate: newStartDate }, }) }) - it('setDisplay reducer updates state correctly', async () => { + it('setCountsEndDate reducer updates state correctly', () => { + const newEndDate = new Date('2024-01-20T12:00:00.000Z') + expect(reducer(initialState, setCountsEndDate(newEndDate))).toEqual({ + ...initialState, + value: { ...initialState.value, countsEndDate: newEndDate }, + }) + }) + + it('setDisplay reducer updates displayCounts correctly', () => { expect(reducer(initialState, setDisplay('displayCounts'))).toEqual({ ...initialState, - value: { ...initialState.value, displayCounts: true }, + value: { + ...initialState.value, + displayCounts: true, + displayRsuErrors: false, + }, + }) + }) + + it('setDisplay reducer updates displayRsuErrors correctly', () => { + expect(reducer(initialState, setDisplay('displayRsuErrors'))).toEqual({ + ...initialState, + value: { + ...initialState.value, + displayCounts: false, + displayRsuErrors: true, + }, }) + }) + it('setDisplay reducer sets both to false for other values', () => { expect(reducer(initialState, setDisplay('somethingElse'))).toEqual({ ...initialState, - value: { ...initialState.value, displayCounts: false }, + value: { + ...initialState.value, + displayCounts: false, + displayRsuErrors: false, + }, + }) + + expect(reducer(initialState, setDisplay(null))).toEqual({ + ...initialState, + value: { + ...initialState.value, + displayCounts: false, + displayRsuErrors: false, + }, }) }) }) -describe('functions', () => { - it('sortCountList ascending', async () => { - const dispatch = jest.fn() - const key = 'key' - const currentSort = 'keydesc' - const countList = [{ key: 1 }, { key: 2 }] as any - const resp = sortCountList(key, currentSort, countList)(dispatch) - expect(resp).toEqual(countList) - expect(dispatch).toHaveBeenCalledTimes(2) - }) - - it('sortCountList descending', async () => { - const dispatch = jest.fn() - const key = 'key' - const currentSort = 'key' - const countList = [{ key: 1 }, { key: 2 }] as any - const resp = sortCountList(key, currentSort, countList)(dispatch) - expect(resp).toEqual([{ key: 2 }, { key: 1 }]) - expect(dispatch).toHaveBeenCalledTimes(2) - }) - - it('changeDate start', async () => { - const dispatch = jest.fn() - const e = DateTime.fromISO('2021-01-01T07:00:00.000Z').toJSDate() - const expected = { start: '2021-01-01T00:00:00.000-07:00' } - const type = 'start' - const requestOut = true - apiHelper._deleteData = jest.fn().mockReturnValue({ status: 200, data: 'data' }) - const resp = changeDate(e, type, requestOut)(dispatch) - expect(resp).toEqual(expected) - expect(dispatch).toHaveBeenCalledTimes(1) - }) - - it('changeDate end', async () => { - const dispatch = jest.fn() - const e = DateTime.fromISO('2021-01-01T07:00:00.000Z').toJSDate() - const expected = { end: '2021-01-01T00:00:00.000-07:00' } - const type = 'end' - const requestOut = true - apiHelper._deleteData = jest.fn().mockReturnValue({ status: 200, data: 'data' }) - const resp = changeDate(e, type, requestOut)(dispatch) - expect(resp).toEqual(expected) - expect(dispatch).toHaveBeenCalledTimes(1) +describe('thunks', () => { + let store: ReturnType + + beforeEach(() => { + store = configureStore({ + reducer: { + menu: reducer, + }, + }) + }) + + describe('toggleMapMenuSelection', () => { + it('adds "Display Message Counts" to menu selection and sets displayCounts', async () => { + const result = await store.dispatch( + toggleMapMenuSelection('Display Message Counts') + ) + + expect(result.payload).toEqual(['Display Message Counts']) + + const state = store.getState() as RootState + expect(state.menu.value.menuSelection).toEqual(['Display Message Counts']) + expect(state.menu.value.displayCounts).toBe(true) + expect(state.menu.value.displayRsuErrors).toBe(false) + }) + + it('adds "Display RSU Status" to menu selection and sets displayRsuErrors', async () => { + const result = await store.dispatch( + toggleMapMenuSelection('Display RSU Status') + ) + + expect(result.payload).toEqual(['Display RSU Status']) + + const state = store.getState() as RootState + expect(state.menu.value.menuSelection).toEqual(['Display RSU Status']) + expect(state.menu.value.displayCounts).toBe(false) + expect(state.menu.value.displayRsuErrors).toBe(true) + }) + + it('removes "Display Message Counts" from menu selection', async () => { + // First add it + await store.dispatch(toggleMapMenuSelection('Display Message Counts')) + + // Then remove it + const result = await store.dispatch( + toggleMapMenuSelection('Display Message Counts') + ) + + expect(result.payload).toEqual([]) + + const state = store.getState() as RootState + expect(state.menu.value.menuSelection).toEqual([]) + expect(state.menu.value.displayCounts).toBe(false) + expect(state.menu.value.displayRsuErrors).toBe(false) + }) + + it('switches from "Display RSU Status" to "Display Message Counts"', async () => { + // First add RSU Status + await store.dispatch(toggleMapMenuSelection('Display RSU Status')) + + // Then toggle Message Counts (should replace RSU Status) + const result = await store.dispatch( + toggleMapMenuSelection('Display Message Counts') + ) + + expect(result.payload).toEqual(['Display Message Counts']) + + const state = store.getState() as RootState + expect(state.menu.value.menuSelection).toEqual(['Display Message Counts']) + expect(state.menu.value.displayCounts).toBe(true) + expect(state.menu.value.displayRsuErrors).toBe(false) + }) + + it('switches from "Display Message Counts" to "Display RSU Status"', async () => { + // First add Message Counts + await store.dispatch(toggleMapMenuSelection('Display Message Counts')) + + // Then toggle RSU Status (should replace Message Counts) + const result = await store.dispatch( + toggleMapMenuSelection('Display RSU Status') + ) + + expect(result.payload).toEqual(['Display RSU Status']) + + const state = store.getState() as RootState + expect(state.menu.value.menuSelection).toEqual(['Display RSU Status']) + expect(state.menu.value.displayCounts).toBe(false) + expect(state.menu.value.displayRsuErrors).toBe(true) + }) + + it('handles adding other menu items without affecting displays', async () => { + const result = await store.dispatch( + toggleMapMenuSelection('Some Other Item') + ) + + expect(result.payload).toEqual(['Some Other Item']) + + const state = store.getState() as RootState + expect(state.menu.value.menuSelection).toEqual(['Some Other Item']) + expect(state.menu.value.displayCounts).toBe(false) + expect(state.menu.value.displayRsuErrors).toBe(false) + }) + + it('can toggle other menu items on and off', async () => { + // Add item + await store.dispatch(toggleMapMenuSelection('Some Other Item')) + let state = store.getState() as RootState + expect(state.menu.value.menuSelection).toEqual(['Some Other Item']) + + // Remove item + await store.dispatch(toggleMapMenuSelection('Some Other Item')) + state = store.getState() as RootState + expect(state.menu.value.menuSelection).toEqual([]) + }) + + it('maintains other selections when toggling display items', async () => { + // Add other item first + await store.dispatch(toggleMapMenuSelection('Other Item')) + + // Add Display Message Counts + const result = await store.dispatch( + toggleMapMenuSelection('Display Message Counts') + ) + + expect(result.payload).toContain('Other Item') + expect(result.payload).toContain('Display Message Counts') + + const state = store.getState() as RootState + expect(state.menu.value.menuSelection.length).toBe(2) + }) }) }) describe('selectors', () => { - const initialState = { - loading: 'loading', - value: { - display: 'display', - currentSort: 'currentSort', - sortedCountList: 'sortedCountList', - displayCounts: 'displayCounts', + const mockStartDate = new Date('2024-01-14T12:00:00.000Z') + const mockEndDate = new Date('2024-01-15T12:00:00.000Z') + + const initialState: RootState = { + menu: { + loading: true, + value: { + countsMsgType: 'SPaT', + countsStartDate: mockStartDate, + countsEndDate: mockEndDate, + displayCounts: true, + displayRsuErrors: false, + menuSelection: ['Display Message Counts', 'Other Item'], + }, }, - } - const state = { menu: initialState } as any + } as RootState + + it('selectLoading returns the correct value', () => { + expect(selectLoading(initialState)).toBe(true) + + const falseState = { + ...initialState, + menu: { ...initialState.menu, loading: false }, + } + expect(selectLoading(falseState)).toBe(false) + }) + + it('selectCountsMsgType returns the correct value', () => { + expect(selectCountsMsgType(initialState)).toBe('SPaT') + + const bsmState = { + ...initialState, + menu: { + ...initialState.menu, + value: { ...initialState.menu.value, countsMsgType: 'BSM' as any }, + }, + } + expect(selectCountsMsgType(bsmState)).toBe('BSM') + }) + + it('selectDisplayCounts returns the correct value', () => { + expect(selectDisplayCounts(initialState)).toBe(true) + + const falseState = { + ...initialState, + menu: { + ...initialState.menu, + value: { ...initialState.menu.value, displayCounts: false }, + }, + } + expect(selectDisplayCounts(falseState)).toBe(false) + }) + + it('selectDisplayRsuErrors returns the correct value', () => { + expect(selectDisplayRsuErrors(initialState)).toBe(false) + + const trueState = { + ...initialState, + menu: { + ...initialState.menu, + value: { ...initialState.menu.value, displayRsuErrors: true }, + }, + } + expect(selectDisplayRsuErrors(trueState)).toBe(true) + }) + + it('selectMenuSelection returns the correct value', () => { + expect(selectMenuSelection(initialState)).toEqual([ + 'Display Message Counts', + 'Other Item', + ]) + + const emptyState = { + ...initialState, + menu: { + ...initialState.menu, + value: { ...initialState.menu.value, menuSelection: [] }, + }, + } + expect(selectMenuSelection(emptyState)).toEqual([]) + }) + + it('selectCountsStartDate returns the correct value', () => { + expect(selectCountsStartDate(initialState)).toEqual(mockStartDate) + }) + + it('selectCountsEndDate returns the correct value', () => { + expect(selectCountsEndDate(initialState)).toEqual(mockEndDate) + }) + + it('all selectors handle undefined state gracefully', () => { + const undefinedState = { + menu: { + loading: false, + value: { + countsMsgType: undefined, + countsStartDate: undefined, + countsEndDate: undefined, + displayCounts: undefined, + displayRsuErrors: undefined, + menuSelection: undefined, + }, + }, + } as any - it('selectors return the correct value', async () => { - expect(selectLoading(state)).toEqual('loading') - expect(selectCurrentSort(state)).toEqual('currentSort') - expect(selectSortedCountList(state)).toEqual('sortedCountList') - expect(selectDisplayCounts(state)).toEqual('displayCounts') + // Selectors should not throw, but return undefined values + expect(() => selectLoading(undefinedState)).not.toThrow() + expect(() => selectCountsMsgType(undefinedState)).not.toThrow() + expect(() => selectDisplayCounts(undefinedState)).not.toThrow() + expect(() => selectDisplayRsuErrors(undefinedState)).not.toThrow() + expect(() => selectMenuSelection(undefinedState)).not.toThrow() + expect(() => selectCountsStartDate(undefinedState)).not.toThrow() + expect(() => selectCountsEndDate(undefinedState)).not.toThrow() }) }) diff --git a/webapp/src/features/menu/menuSlice.tsx b/webapp/src/features/menu/menuSlice.tsx index 3d933ce1d..4d4c5087a 100644 --- a/webapp/src/features/menu/menuSlice.tsx +++ b/webapp/src/features/menu/menuSlice.tsx @@ -1,67 +1,22 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ -import { AnyAction, PayloadAction, ThunkDispatch, createAsyncThunk, createSlice } from '@reduxjs/toolkit' -import { updateRowData } from '../../generalSlices/rsuSlice' +import { PayloadAction, createAsyncThunk, createSlice } from '@reduxjs/toolkit' import { RootState } from '../../store' -import { CountsListElement } from '../../models/Rsu' -import { DateTime } from 'luxon' +import { MessageType } from '../../models/RsuApi' +import dayjs from 'dayjs' const initialState = { - currentSort: null as null | string, - sortedCountList: [] as CountsListElement[], + countsMsgType: 'BSM' as MessageType, + countsStartDate: dayjs().subtract(1, 'day').toDate(), + countsEndDate: dayjs().toDate(), displayCounts: false, displayRsuErrors: false, menuSelection: [], } -export const sortCountList = - (key: string, currentSort: string, countList: CountsListElement[]) => - (dispatch: ThunkDispatch) => { - let sortFn = ( - a: { [key: string]: string | number | void }, - b: { [key: string]: string | number | void } - ): number => { - return 0 - } - // Support both descending and ascending sort - // based on the current sort - // Default is ascending - if (key === currentSort) { - dispatch(setCurrentSort(key + 'desc')) - sortFn = function (a, b) { - if (a[key] > b[key]) return -1 - if (a[key] < b[key]) return 1 - return 0 - } - } else { - dispatch(setCurrentSort(key)) - sortFn = function (a, b) { - if (a[key] < b[key]) return -1 - if (a[key] > b[key]) return 1 - return 0 - } - } - - const arrayCopy = [...countList] - arrayCopy.sort(sortFn) - dispatch(setSortedCountList(arrayCopy)) - return arrayCopy - } - -export const changeDate = (e: Date, type: 'start' | 'end') => (dispatch: ThunkDispatch) => { - const mst = DateTime.fromJSDate(e).setZone('America/Denver') - let data - if (type === 'start') { - data = { start: mst.toString() } - } else { - data = { end: mst.toString() } - } - dispatch(updateRowData(data)) - return data -} - export const toggleMapMenuSelection = createAsyncThunk( 'menu/toggleMapMenuSelection', async (label: string, { getState, dispatch }) => { + // TODO: Re-factor this to not store list of menuSelection. This should be broken out into separate menu variables if needed const currentState = getState() as RootState let menuSelection = [...selectMenuSelection(currentState)] if (menuSelection.includes(label)) { @@ -78,13 +33,13 @@ export const toggleMapMenuSelection = createAsyncThunk( switch (label) { case 'Display Message Counts': if (menuSelection.includes('Display RSU Status')) { - menuSelection = [...menuSelection.filter((item) => item !== 'Display RSU Status'), 'Display Message Counts'] + menuSelection = [...menuSelection.filter((item) => item !== 'Display RSU Status')] } dispatch(setDisplay('displayCounts')) break case 'Display RSU Status': if (menuSelection.includes('Display Message Counts')) { - menuSelection = [...menuSelection.filter((item) => item !== 'Display Message Counts'), 'Display RSU Status'] + menuSelection = [...menuSelection.filter((item) => item !== 'Display Message Counts')] } dispatch(setDisplay('displayRsuErrors')) } @@ -100,16 +55,22 @@ export const menuSlice = createSlice({ value: initialState, }, reducers: { - setCurrentSort: (state, action) => { - state.value.currentSort = action.payload + setCountsMsgType: (state, action: PayloadAction) => { + state.value.countsMsgType = action.payload }, - setSortedCountList: (state, action) => { - state.value.sortedCountList = action.payload + setCountsStartDate: (state, action: PayloadAction) => { + state.value.countsStartDate = action.payload + }, + setCountsEndDate: (state, action: PayloadAction) => { + state.value.countsEndDate = action.payload }, setDisplay: (state, action: PayloadAction) => { state.value.displayCounts = action.payload == 'displayCounts' state.value.displayRsuErrors = action.payload == 'displayRsuErrors' }, + setMenuSelection: (state, action: PayloadAction) => { + state.value.menuSelection = action.payload + }, }, extraReducers: (builder) => { builder.addCase(toggleMapMenuSelection.fulfilled, (state, action) => { @@ -118,13 +79,14 @@ export const menuSlice = createSlice({ }, }) -export const { setCurrentSort, setSortedCountList, setDisplay } = menuSlice.actions +export const { setCountsMsgType, setCountsStartDate, setCountsEndDate, setDisplay } = menuSlice.actions export const selectLoading = (state: RootState) => state.menu.loading -export const selectCurrentSort = (state: RootState) => state.menu.value.currentSort -export const selectSortedCountList = (state: RootState) => state.menu.value.sortedCountList +export const selectCountsMsgType = (state: RootState) => state.menu.value.countsMsgType export const selectDisplayCounts = (state: RootState) => state.menu.value.displayCounts export const selectDisplayRsuErrors = (state: RootState) => state.menu.value.displayRsuErrors export const selectMenuSelection = (state: RootState) => state.menu.value.menuSelection +export const selectCountsStartDate = (state: RootState) => state.menu.value.countsStartDate +export const selectCountsEndDate = (state: RootState) => state.menu.value.countsEndDate export default menuSlice.reducer diff --git a/webapp/src/generalSlices/rsuSlice.test.ts b/webapp/src/generalSlices/rsuSlice.test.ts index 09df97127..718735aab 100644 --- a/webapp/src/generalSlices/rsuSlice.test.ts +++ b/webapp/src/generalSlices/rsuSlice.test.ts @@ -2,19 +2,13 @@ import reducer from './rsuSlice' import { // async thunks getRsuData, - getRsuInfoOnly, getRsuLastOnline, _getRsuInfo, _getRsuOnlineStatus, - _getRsuCounts, getSsmSrmData, getIssScmsStatus, - updateRowData, updateGeoMsgData, - // functions - updateMessageType, - // reducers selectRsu, toggleMapDisplay, @@ -25,7 +19,6 @@ import { updateGeoMsgPoints, updateGeoMsgDate, triggerGeoMsgDateError, - changeCountsMsgType, setGeoMsgFilter, setGeoMsgFilterStep, setGeoMsgFilterOffset, @@ -33,21 +26,12 @@ import { // selectors selectLoading, - selectRequestOut, selectSelectedRsu, selectRsuManufacturer, selectRsuIpv4, selectRsuPrimaryRoute, selectRsuData, selectRsuOnlineStatus, - selectRsuCounts, - selectCountList, - selectCurrentSort, - selectStartDate, - selectEndDate, - selectMessageLoading, - selectWarningMessage, - selectMsgType, selectRsuMapData, selectMapList, selectMapDate, @@ -65,7 +49,6 @@ import { selectSsmDisplay, selectSrmSsmList, selectSelectedSrm, - selectHeatMapData, } from './rsuSlice' import RsuApi from '../apis/rsu-api' import { RootState } from '../store' @@ -89,23 +72,10 @@ describe('rsu reducer', () => { it('should handle initial state', () => { expect(reducer(undefined, { type: 'unknown' })).toEqual({ loading: false, - requestOut: false, value: { selectedRsu: null, rsuData: [], rsuOnlineStatus: {}, - rsuCounts: {}, - countList: [], - currentSort: '', - startDate: currentDate.minus({ days: 1 }).toString(), - endDate: currentDate.toString(), - heatMapData: { - features: [], - type: 'FeatureCollection', - }, - messageLoading: false, - warningMessage: false, - countsMsgType: 'BSM', rsuMapData: {}, mapList: [], mapDate: '', @@ -132,22 +102,10 @@ describe('rsu reducer', () => { describe('async thunks', () => { const initialState: RootState['rsu'] = { loading: null, - requestOut: null, value: { selectedRsu: null, rsuData: null, rsuOnlineStatus: null, - rsuCounts: null, - countList: null, - currentSort: null, - startDate: null, - endDate: null, - heatMapData: { - features: [], - type: 'FeatureCollection', - }, - messageLoading: null, - warningMessage: null, countsMsgType: null, geoMsgType: null, rsuMapData: null, @@ -191,23 +149,19 @@ describe('async thunks', () => { rsu: { value: { rsuOnlineStatus: {}, - startDate: '', - endDate: '', }, }, }) const action = getRsuData() await action(dispatch, getState, undefined) - expect(dispatch).toHaveBeenCalledTimes(4 + 2) // 4 for the 4 dispatched actions, 2 for the pending and fulfilled actions + expect(dispatch).toHaveBeenCalledTimes(2 + 2) // 2 for the 2 dispatched actions, 2 for the pending and fulfilled actions }) it('Updates the state correctly pending', async () => { const loading = true const rsuData = [] as any const rsuOnlineStatus = {} - const rsuCounts = {} - const countList = [] as any const state = reducer(initialState, { type: 'rsu/getRsuData/pending', }) @@ -218,15 +172,12 @@ describe('async thunks', () => { ...initialState.value, rsuData, rsuOnlineStatus, - rsuCounts, - countList, }, }) }) it('Updates the state correctly fulfilled', async () => { const loading = false - const rsuCounts = { ipv4_address: { count: 4 } } as any const rsuData = [ { properties: { @@ -238,32 +189,16 @@ describe('async thunks', () => { }, ] as any const state = reducer( - { ...initialState, value: { ...initialState.value, rsuData, rsuCounts } }, + { ...initialState, value: { ...initialState.value, rsuData } }, { type: 'rsu/getRsuData/fulfilled', } ) - const heatMapData = { - features: [ - { - type: 'Feature', - geometry: { - type: 'Point', - coordinates: [-104.999824, 39.750392], - }, - properties: { - ipv4_address: 'ipv4_address', - count: 4, - }, - }, - ], - type: 'FeatureCollection', - } expect(state).toEqual({ ...initialState, loading, - value: { ...initialState.value, rsuData, rsuCounts, heatMapData }, + value: { ...initialState.value, rsuData }, }) }) @@ -276,55 +211,6 @@ describe('async thunks', () => { }) }) - describe('getRsuInfoOnly', () => { - it('returns and calls the api correctly', async () => { - const dispatch = jest.fn() - const getState = jest.fn().mockReturnValue({ - user: { - value: { - authLoginData: { token: 'token' }, - organization: { name: 'name' }, - }, - }, - }) - const action = getRsuInfoOnly() - - const rsuData = ['1.1.1.1'] - RsuApi.getRsuInfo = jest.fn().mockReturnValue({ rsuList: rsuData }) - const resp = await action(dispatch, getState, undefined) - expect(resp.payload).toEqual(rsuData) - expect(RsuApi.getRsuInfo).toHaveBeenCalledWith('token', 'name') - }) - - it('Updates the state correctly pending', async () => { - const loading = true - const state = reducer(initialState, { - type: 'rsu/getRsuInfoOnly/pending', - }) - expect(state).toEqual({ - ...initialState, - loading, - value: { ...initialState.value }, - }) - }) - - it('Updates the state correctly fulfilled', async () => { - const loading = false - const state = reducer(initialState, { - type: 'rsu/getRsuInfoOnly/fulfilled', - }) - expect(state).toEqual({ ...initialState, loading, value: { ...initialState.value } }) - }) - - it('Updates the state correctly rejected', async () => { - const loading = false - const state = reducer(initialState, { - type: 'rsu/getRsuInfoOnly/rejected', - }) - expect(state).toEqual({ ...initialState, loading, value: { ...initialState.value } }) - }) - }) - describe('getRsuLastOnline', () => { it('returns and calls the api correctly', async () => { const dispatch = jest.fn() @@ -470,104 +356,6 @@ describe('async thunks', () => { }) }) - describe('_getRsuCounts', () => { - it('returns and calls the api correctly', async () => { - const dispatch = jest.fn() - const getState = jest.fn().mockReturnValue({ - user: { - value: { - authLoginData: { token: 'token' }, - organization: { name: 'name' }, - }, - }, - rsu: { - value: { - countsMsgType: 'BSM', - startDate: '', - endDate: '', - }, - }, - }) - const action = _getRsuCounts() - - const rsuCounts = { - '1.1.1.1': { road: 'road', count: 'count' }, - } - const countList = [ - { - key: '1.1.1.1', - rsu: '1.1.1.1', - road: 'road', - count: 'count', - }, - ] - RsuApi.getRsuCounts = jest.fn().mockReturnValue(rsuCounts) - const resp = await action(dispatch, getState, undefined) - expect(resp.payload).toEqual({ rsuCounts, countList }) - expect(RsuApi.getRsuCounts).toHaveBeenCalledWith('token', 'name', '', { - message: 'BSM', - start: '', - end: '', - }) - }) - it('returns and calls the api correctly', async () => { - const rsuCounts = { - '1.1.1.1': { road: 'road', count: 'count' }, - } - - const dispatch = jest.fn() - const getState = jest.fn().mockReturnValue({ - user: { - value: { - authLoginData: { token: 'token' }, - organization: { name: 'name' }, - }, - }, - rsu: { - value: { - countsMsgType: 'BSM', - startDate: '', - endDate: '', - rsuCounts, - }, - }, - }) - - const action = _getRsuCounts() - const countList = [ - { - key: '1.1.1.1', - rsu: '1.1.1.1', - road: 'road', - count: 'count', - }, - ] - RsuApi.getRsuCounts = jest.fn().mockReturnValue(null) - const resp = await action(dispatch, getState, undefined) - expect(resp.payload).toEqual({ rsuCounts, countList }) - expect(RsuApi.getRsuCounts).toHaveBeenCalledWith('token', 'name', '', { - message: 'BSM', - start: '', - end: '', - }) - }) - - it('Updates the state correctly fulfilled', async () => { - const rsuCounts = 'rsuCounts' - const countList = 'countList' - const payload = { rsuCounts, countList } - const state = reducer(initialState, { - type: 'rsu/_getRsuCounts/fulfilled', - payload: payload, - }) - - expect(state).toEqual({ - ...initialState, - value: { ...initialState.value, rsuCounts, countList }, - }) - }) - }) - describe('getSsmSrmData', () => { it('returns and calls the api correctly', async () => { const dispatch = jest.fn() @@ -649,191 +437,6 @@ describe('async thunks', () => { }) }) - describe('updateRowData', () => { - it('returns and calls the api correctly', async () => { - const dispatch = jest.fn() - const getState = jest.fn().mockReturnValue({ - user: { - value: { - authLoginData: { token: 'token' }, - organization: { name: 'name' }, - }, - }, - }) - const data = { - message: 'message', - start: 1, - end: 86400000, - } - const action = updateRowData(data as any) - - const rsuCounts = { - '1.1.1.1': { road: 'road', count: 'count' }, - } - const countList = [ - { - key: '1.1.1.1', - rsu: '1.1.1.1', - road: 'road', - count: 'count', - }, - ] - RsuApi.getRsuCounts = jest.fn().mockReturnValue(rsuCounts) - const resp = await action(dispatch, getState, undefined) - expect(resp.payload).toEqual({ - countsMsgType: 'message', - startDate: 1, - endDate: 86400000, - warningMessage: false, - rsuCounts, - countList, - }) - expect(RsuApi.getRsuCounts).toHaveBeenCalledWith('token', 'name', '', data) - }) - - it('returns and calls the api correctly default values', async () => { - const dispatch = jest.fn() - const getState = jest.fn().mockReturnValue({ - user: { - value: { - authLoginData: { token: 'token' }, - organization: { name: 'name' }, - }, - }, - rsu: { - value: { - countsMsgType: 'message', - startDate: 1, - endDate: 86400002, - }, - }, - }) - const data = {} - const action = updateRowData(data) - - const rsuCounts = { - '1.1.1.1': { road: 'road', count: 'count' }, - } - const countList = [ - { - key: '1.1.1.1', - rsu: '1.1.1.1', - road: 'road', - count: 'count', - }, - ] - RsuApi.getRsuCounts = jest.fn().mockReturnValue(rsuCounts) - const resp = await action(dispatch, getState, undefined) - expect(resp.payload).toEqual({ - countsMsgType: 'message', - startDate: 1, - endDate: 86400002, - warningMessage: true, - rsuCounts, - countList, - }) - expect(RsuApi.getRsuCounts).toHaveBeenCalledWith('token', 'name', '', { - message: 'message', - start: 1, - end: 86400002, - }) - }) - - it('Updates the state correctly pending', async () => { - const requestOut = true - const messageLoading = false - const state = reducer(initialState, { - type: 'rsu/updateRowData/pending', - }) - - expect(state).toEqual({ - ...initialState, - requestOut, - value: { ...initialState.value, messageLoading }, - }) - }) - - it('Updates the state correctly fulfilled', async () => { - const rsuCounts = { '1.1.1.1': { count: 5 } } - const countList = 'countList' - const heatMapData = { - type: 'FeatureCollection', - features: [ - { - properties: { - ipv4_address: '1.1.1.1', - }, - }, - { - properties: { - ipv4_address: '1.1.1.2', - }, - }, - ], - } as any - const warningMessage = 'warningMessage' - const requestOut = false - const messageLoading = false - const countsMsgType = 'countsMsgType' - const startDate = 'startDate' - const endDate = 'endDate' - const payload = { - rsuCounts, - countList, - warningMessage, - countsMsgType, - startDate, - endDate, - } - const state = reducer( - { - ...initialState, - value: { - ...initialState.value, - heatMapData, - }, - }, - { - type: 'rsu/updateRowData/fulfilled', - payload: payload, - } - ) - - heatMapData['features'][0]['properties']['count'] = 5 - heatMapData['features'][1]['properties']['count'] = 0 - - expect(state).toEqual({ - ...initialState, - requestOut, - value: { - ...initialState.value, - rsuCounts, - countList, - heatMapData, - warningMessage, - messageLoading, - countsMsgType, - startDate, - endDate, - }, - }) - }) - - it('Updates the state correctly rejected', async () => { - const requestOut = false - const messageLoading = false - const state = reducer(initialState, { - type: 'rsu/updateRowData/rejected', - }) - - expect(state).toEqual({ - ...initialState, - requestOut, - value: { ...initialState.value, messageLoading }, - }) - }) - }) - describe('updateGeoMsgData', () => { it('returns and calls the api correctly', async () => { const dispatch = jest.fn() @@ -976,35 +579,13 @@ describe('async thunks', () => { }) }) -describe('functions', () => { - it('updateMessageType', async () => { - const dispatch = jest.fn() - - updateMessageType('messageType' as any)(dispatch) - expect(dispatch).toHaveBeenCalledTimes(2) - }) -}) - describe('reducers', () => { const initialState: RootState['rsu'] = { loading: null, - requestOut: null, value: { selectedRsu: null, rsuData: null, rsuOnlineStatus: null, - rsuCounts: null, - countList: null, - currentSort: null, - startDate: null, - endDate: null, - heatMapData: { - features: [], - type: 'FeatureCollection', - }, - messageLoading: null, - warningMessage: null, - countsMsgType: null, geoMsgType: null, rsuMapData: null, mapList: null, @@ -1133,14 +714,6 @@ describe('reducers', () => { }) }) - it('changeCountsMsgType reducer updates state correctly', async () => { - const countsMsgType = 'countsMsgType' - expect(reducer(initialState, changeCountsMsgType(countsMsgType))).toEqual({ - ...initialState, - value: { ...initialState.value, countsMsgType }, - }) - }) - it('setGeoMsgFilter reducer updates state correctly', async () => { const geoMsgFilter = true expect(reducer(initialState, setGeoMsgFilter(geoMsgFilter))).toEqual({ @@ -1178,7 +751,6 @@ describe('reducers', () => { describe('selectors', () => { const initialState = { loading: 'loading', - requestOut: 'requestOut', value: { selectedRsu: { properties: { @@ -1189,14 +761,6 @@ describe('selectors', () => { }, rsuData: 'rsuData', rsuOnlineStatus: 'rsuOnlineStatus', - rsuCounts: 'rsuCounts', - countList: 'countList', - currentSort: 'currentSort', - startDate: 'startDate', - endDate: 'endDate', - heatMapData: 'heatMapData', - messageLoading: 'messageLoading', - warningMessage: 'warningMessage', countsMsgType: 'countsMsgType', rsuMapData: 'rsuMapData', mapList: 'mapList', @@ -1221,7 +785,6 @@ describe('selectors', () => { it('selectors return the correct value', async () => { expect(selectLoading(rsuState)).toEqual('loading') - expect(selectRequestOut(rsuState)).toEqual('requestOut') expect(selectSelectedRsu(rsuState)).toEqual(initialState.value.selectedRsu) expect(selectRsuManufacturer(rsuState)).toEqual('manufacturer_name') @@ -1229,14 +792,6 @@ describe('selectors', () => { expect(selectRsuPrimaryRoute(rsuState)).toEqual('primary_route') expect(selectRsuData(rsuState)).toEqual('rsuData') expect(selectRsuOnlineStatus(rsuState)).toEqual('rsuOnlineStatus') - expect(selectRsuCounts(rsuState)).toEqual('rsuCounts') - expect(selectCountList(rsuState)).toEqual('countList') - expect(selectCurrentSort(rsuState)).toEqual('currentSort') - expect(selectStartDate(rsuState)).toEqual('startDate') - expect(selectEndDate(rsuState)).toEqual('endDate') - expect(selectMessageLoading(rsuState)).toEqual('messageLoading') - expect(selectWarningMessage(rsuState)).toEqual('warningMessage') - expect(selectMsgType(rsuState)).toEqual('countsMsgType') expect(selectRsuMapData(rsuState)).toEqual('rsuMapData') expect(selectMapList(rsuState)).toEqual('mapList') expect(selectMapDate(rsuState)).toEqual('mapDate') @@ -1254,6 +809,5 @@ describe('selectors', () => { expect(selectSsmDisplay(rsuState)).toEqual('ssmDisplay') expect(selectSrmSsmList(rsuState)).toEqual('srmSsmList') expect(selectSelectedSrm(rsuState)).toEqual('selectedSrm') - expect(selectHeatMapData(rsuState)).toEqual('heatMapData') }) }) diff --git a/webapp/src/generalSlices/rsuSlice.ts b/webapp/src/generalSlices/rsuSlice.ts index 6c0fa9805..eb9f98539 100644 --- a/webapp/src/generalSlices/rsuSlice.ts +++ b/webapp/src/generalSlices/rsuSlice.ts @@ -1,8 +1,7 @@ -import { AnyAction, createAsyncThunk, createSlice, PayloadAction, ThunkDispatch } from '@reduxjs/toolkit' +import { createAsyncThunk, createSlice, PayloadAction } from '@reduxjs/toolkit' import RsuApi from '../apis/rsu-api' import { IssScmsStatus, - RsuCounts, RsuInfo, RsuMapInfo, RsuMapInfoIpList, @@ -13,7 +12,6 @@ import { import { RootState } from '../store' import { selectToken, selectOrganizationName } from './userSlice' import { SelectedSrm } from '../models/Srm' -import { CountsListElement } from '../models/Rsu' import { MessageType } from '../models/MessageTypes' import { toast } from 'react-hot-toast' import { DateTime } from 'luxon' @@ -24,15 +22,7 @@ const initialState = { selectedRsu: null as RsuInfo, rsuData: [] as RsuInfo[], rsuOnlineStatus: {} as RsuOnlineStatusRespMultiple, - rsuCounts: {} as RsuCounts, - countList: [] as CountsListElement[], - currentSort: '', - startDate: currentDate.minus({ days: 1 }).toString(), - endDate: currentDate.toString(), - messageLoading: false, - warningMessage: false, - countsMsgType: 'BSM', - geoMsgType: 'BSM', + geoMsgType: 'BSM' as MessageType | undefined, rsuMapData: {} as RsuMapInfo['geojson'], mapList: [] as RsuMapInfoIpList, mapDate: '' as RsuMapInfo['date'], @@ -52,44 +42,20 @@ const initialState = { ssmDisplay: false, srmSsmList: [] as SsmSrmData, selectedSrm: [] as SelectedSrm[], - heatMapData: { - type: 'FeatureCollection', - features: [], - } as GeoJSON.FeatureCollection, } -export const updateMessageType = - (messageType: MessageType) => async (dispatch: ThunkDispatch) => { - dispatch(changeCountsMsgType(messageType)) - dispatch(updateRowData({ message: messageType })) - } - export const getRsuData = createAsyncThunk( 'rsu/getRsuData', async (_, { getState, dispatch }) => { const currentState = getState() as RootState - await Promise.all([ - dispatch(resetCountsDates()), - dispatch(_getRsuInfo()), - dispatch(_getRsuOnlineStatus(currentState.rsu.value.rsuOnlineStatus)), - dispatch(_getRsuCounts()), - ]) + await Promise.all([dispatch(_getRsuInfo()), dispatch(_getRsuOnlineStatus(currentState.rsu.value.rsuOnlineStatus))]) }, { condition: (_, { getState }) => selectToken(getState() as RootState) != undefined, } ) -export const getRsuInfoOnly = createAsyncThunk('rsu/getRsuInfoOnly', async (_, { getState }) => { - const currentState = getState() as RootState - const token = selectToken(currentState) - const organization = selectOrganizationName(currentState) - const rsuInfo = await RsuApi.getRsuInfo(token, organization) - const rsuData = rsuInfo.rsuList - return rsuData -}) - export const getRsuLastOnline = createAsyncThunk('rsu/getRsuLastOnline', async (rsu_ip: string, { getState }) => { const currentState = getState() as RootState const token = selectToken(currentState) @@ -120,30 +86,6 @@ export const _getRsuOnlineStatus = createAsyncThunk( } ) -export const _getRsuCounts = createAsyncThunk('rsu/_getRsuCounts', async (_, { getState }) => { - const currentState = getState() as RootState - const token = selectToken(currentState) - const organization = selectOrganizationName(currentState) - - const query_params = { - message: currentState.rsu.value.countsMsgType, - start: currentState.rsu.value.startDate, - end: currentState.rsu.value.endDate, - } - const rsuCounts = - (await RsuApi.getRsuCounts(token, organization, '', query_params)) ?? currentState.rsu.value.rsuCounts - const countList = Object.entries(rsuCounts).map(([key, value]) => { - return { - key: key, - rsu: key, - road: value.road, - count: value.count, - } - }) - - return { rsuCounts, countList } -}) - export const getSsmSrmData = createAsyncThunk('rsu/getSsmSrmData', async (_, { getState }) => { const currentState = getState() as RootState const token = selectToken(currentState) @@ -164,59 +106,6 @@ export const getIssScmsStatus = createAsyncThunk( } ) -export const updateRowData = createAsyncThunk( - 'rsu/updateRowData', - async ( - data: { - message?: MessageType - start?: string - end?: string - }, - { getState } - ) => { - const currentState = getState() as RootState - const token = selectToken(currentState) - const organization = selectOrganizationName(currentState) - - const countsMsgType = Object.prototype.hasOwnProperty.call(data, 'message') - ? data['message'] - : currentState.rsu.value.countsMsgType - const startDate = Object.prototype.hasOwnProperty.call(data, 'start') - ? data['start'] - : currentState.rsu.value.startDate - const endDate = Object.prototype.hasOwnProperty.call(data, 'end') ? data['end'] : currentState.rsu.value.endDate - - const warningMessage = new Date(endDate).getTime() - new Date(startDate).getTime() > 86400000 - - const rsuCountsData = await RsuApi.getRsuCounts(token, organization, '', { - message: countsMsgType, - start: startDate, - end: endDate, - }) - - const countList = Object.entries(rsuCountsData).map(([key, value]) => { - return { - key: key, - rsu: key, - road: value.road, - count: value.count, - } - }) - - return { - countsMsgType, - startDate, - endDate, - warningMessage, - rsuCounts: rsuCountsData, - countList, - } - }, - { - condition: (_, { getState }) => selectToken(getState() as RootState) != undefined, - } -) - export const updateGeoMsgData = createAsyncThunk( 'rsu/updateGeoMsgData', async (_, { getState }) => { @@ -278,11 +167,7 @@ export const updateGeoMsgData = createAsyncThunk( // Will guard thunk from being executed condition: (_, { getState }) => { const { rsu } = getState() as RootState - const valid = - rsu.value.geoMsgStart !== '' && - rsu.value.geoMsgEnd !== '' && - rsu.value.geoMsgCoordinates.length > 2 && - rsu.value.countsMsgType !== '' + const valid = rsu.value.geoMsgStart !== '' && rsu.value.geoMsgEnd !== '' && rsu.value.geoMsgCoordinates.length > 2 return valid }, } @@ -292,7 +177,6 @@ export const rsuSlice = createSlice({ name: 'rsu', initialState: { loading: false, - requestOut: false, value: initialState, }, reducers: { @@ -326,10 +210,7 @@ export const rsuSlice = createSlice({ triggerGeoMsgDateError: (state) => { state.value.geoMsgDateError = true }, - changeCountsMsgType: (state, action) => { - state.value.countsMsgType = action.payload - }, - changeGeoMsgType: (state, action: PayloadAction) => { + changeGeoMsgType: (state, action: PayloadAction) => { state.value.geoMsgType = action.payload }, setGeoMsgFilter: (state, action: PayloadAction) => { @@ -344,11 +225,6 @@ export const rsuSlice = createSlice({ setLoading: (state, action: PayloadAction) => { state.loading = action.payload }, - resetCountsDates: (state) => { - const now = DateTime.local().setZone(DateTime.local().zoneName) - state.value.startDate = now.minus({ days: 1 }).toString() - state.value.endDate = now.toString() - }, }, extraReducers: (builder) => { builder @@ -356,46 +232,13 @@ export const rsuSlice = createSlice({ state.loading = true state.value.rsuData = [] state.value.rsuOnlineStatus = {} - state.value.rsuCounts = {} - state.value.countList = [] - state.value.heatMapData = { - type: 'FeatureCollection', - features: [], - } }) .addCase(getRsuData.fulfilled, (state) => { - const heatMapFeatures: GeoJSON.Feature[] = [] - state.value.rsuData.forEach((rsu) => { - heatMapFeatures.push({ - type: 'Feature', - geometry: { - type: 'Point', - coordinates: [rsu.geometry.coordinates[0], rsu.geometry.coordinates[1]], - }, - properties: { - ipv4_address: rsu.properties.ipv4_address, - count: - rsu.properties.ipv4_address in state.value.rsuCounts - ? state.value.rsuCounts[rsu.properties.ipv4_address].count - : 0, - }, - }) - }) - state.value.heatMapData.features = heatMapFeatures state.loading = false }) .addCase(getRsuData.rejected, (state) => { state.loading = false }) - .addCase(getRsuInfoOnly.pending, (state) => { - state.loading = true - }) - .addCase(getRsuInfoOnly.fulfilled, (state) => { - state.loading = false - }) - .addCase(getRsuInfoOnly.rejected, (state) => { - state.loading = false - }) .addCase(getRsuLastOnline.pending, (state) => { state.loading = true }) @@ -403,7 +246,7 @@ export const rsuSlice = createSlice({ state.loading = false const payload = action.payload as RsuOnlineStatusRespSingle if (Object.prototype.hasOwnProperty.call(state.value.rsuOnlineStatus, payload.ip)) { - (state.value.rsuOnlineStatus as RsuOnlineStatusRespMultiple)[payload.ip]['last_online'] = payload.last_online + ;(state.value.rsuOnlineStatus as RsuOnlineStatusRespMultiple)[payload.ip]['last_online'] = payload.last_online } }) .addCase(getRsuLastOnline.rejected, (state) => { @@ -415,10 +258,6 @@ export const rsuSlice = createSlice({ .addCase(_getRsuOnlineStatus.fulfilled, (state, action) => { state.value.rsuOnlineStatus = action.payload as RsuOnlineStatusRespMultiple }) - .addCase(_getRsuCounts.fulfilled, (state, action) => { - state.value.rsuCounts = action.payload.rsuCounts - state.value.countList = action.payload.countList - }) .addCase(getSsmSrmData.pending, (state) => { state.loading = true }) @@ -431,30 +270,6 @@ export const rsuSlice = createSlice({ .addCase(getIssScmsStatus.fulfilled, (state, action) => { state.value.issScmsStatusData = action.payload ?? state.value.issScmsStatusData }) - .addCase(updateRowData.pending, (state) => { - state.requestOut = true - state.value.messageLoading = false - }) - .addCase(updateRowData.fulfilled, (state, action) => { - if (action.payload === null) return - state.value.rsuCounts = action.payload.rsuCounts - state.value.countList = action.payload.countList - state.value.heatMapData.features.forEach((feat, index) => { - const ip = feat.properties.ipv4_address as string - state.value.heatMapData.features[index].properties.count = - ip in action.payload.rsuCounts ? action.payload.rsuCounts[ip].count : 0 - }) - state.value.warningMessage = action.payload.warningMessage - state.requestOut = false - state.value.messageLoading = false - state.value.countsMsgType = action.payload.countsMsgType - state.value.startDate = action.payload.startDate - state.value.endDate = action.payload.endDate - }) - .addCase(updateRowData.rejected, (state) => { - state.requestOut = false - state.value.messageLoading = false - }) .addCase(updateGeoMsgData.pending, (state) => { state.loading = true state.value.addGeoMsgPoint = false @@ -473,7 +288,6 @@ export const rsuSlice = createSlice({ }) export const selectLoading = (state: RootState) => state.rsu.loading -export const selectRequestOut = (state: RootState) => state.rsu.requestOut export const selectSelectedRsu = (state: RootState) => state.rsu.value.selectedRsu export const selectRsuManufacturer = (state: RootState) => state.rsu.value.selectedRsu?.properties?.manufacturer_name @@ -481,14 +295,6 @@ export const selectRsuIpv4 = (state: RootState) => state.rsu.value.selectedRsu?. export const selectRsuPrimaryRoute = (state: RootState) => state.rsu.value.selectedRsu?.properties?.primary_route export const selectRsuData = (state: RootState) => state.rsu.value.rsuData export const selectRsuOnlineStatus = (state: RootState) => state.rsu.value.rsuOnlineStatus -export const selectRsuCounts = (state: RootState) => state.rsu.value.rsuCounts -export const selectCountList = (state: RootState) => state.rsu.value.countList -export const selectCurrentSort = (state: RootState) => state.rsu.value.currentSort -export const selectStartDate = (state: RootState) => state.rsu.value.startDate -export const selectEndDate = (state: RootState) => state.rsu.value.endDate -export const selectMessageLoading = (state: RootState) => state.rsu.value.messageLoading -export const selectWarningMessage = (state: RootState) => state.rsu.value.warningMessage -export const selectMsgType = (state: RootState) => state.rsu.value.countsMsgType export const selectGeoMsgType = (state: RootState) => state.rsu.value.geoMsgType export const selectRsuMapData = (state: RootState) => state.rsu.value.rsuMapData export const selectMapList = (state: RootState) => state.rsu.value.mapList @@ -507,7 +313,6 @@ export const selectIssScmsStatusData = (state: RootState) => state.rsu.value.iss export const selectSsmDisplay = (state: RootState) => state.rsu.value.ssmDisplay export const selectSrmSsmList = (state: RootState) => state.rsu.value.srmSsmList export const selectSelectedSrm = (state: RootState) => state.rsu.value.selectedSrm -export const selectHeatMapData = (state: RootState) => state.rsu.value.heatMapData export const { selectRsu, @@ -519,13 +324,11 @@ export const { updateGeoMsgPoints, updateGeoMsgDate, triggerGeoMsgDateError, - changeCountsMsgType, changeGeoMsgType, setGeoMsgFilter, setGeoMsgFilterStep, setGeoMsgFilterOffset, setLoading, - resetCountsDates, } = rsuSlice.actions export default rsuSlice.reducer diff --git a/webapp/src/generalSlices/timeSyncSlice.test.ts b/webapp/src/generalSlices/timeSyncSlice.test.ts index 3bea50caf..8d8381589 100644 --- a/webapp/src/generalSlices/timeSyncSlice.test.ts +++ b/webapp/src/generalSlices/timeSyncSlice.test.ts @@ -73,7 +73,8 @@ describe('async thunks', () => { }) it('syncTimeOffset should synchronize time offset (mocked fetch)', async () => { - const mockServerTime = '2025-10-20T21:28:30.0960336' + const mockServerTime = '2025-10-20T21:28:30.0960336Z' + const mockServerTimeMillis = new Date(mockServerTime).getTime() const mockResponse = { year: 2025, month: 10, @@ -93,20 +94,23 @@ describe('async thunks', () => { json: jest.fn().mockResolvedValueOnce(mockResponse), }) + // Mock Date.now() to return the server time + const originalDateNow = Date.now + Date.now = jest.fn(() => mockServerTimeMillis) + const dispatch = jest.fn() const getState = jest.fn() const action = syncTimeOffset() - const start = Date.now() const result = await action(dispatch, getState, undefined) - const end = Date.now() - const serverTime = new Date(mockServerTime).getTime() - const rtt = end - start - const expectedOffset = serverTime + rtt / 2 - Date.now() + const expectedOffset = 0 expect(result.payload).toBeCloseTo(expectedOffset, -2) // Allow slight timing differences expect(global.fetch).toHaveBeenCalledWith(TIME_SERVER_URL_UTC) + + // Restore original Date.now + Date.now = originalDateNow }) }) diff --git a/webapp/src/generalSlices/timeSyncSlice.ts b/webapp/src/generalSlices/timeSyncSlice.ts index cc9d779aa..025dbcdd8 100644 --- a/webapp/src/generalSlices/timeSyncSlice.ts +++ b/webapp/src/generalSlices/timeSyncSlice.ts @@ -1,6 +1,7 @@ // store/timeSyncSlice.ts import { createAsyncThunk, createSlice, PayloadAction } from '@reduxjs/toolkit' import { RootState } from '../store' +import { fromZonedTime } from 'date-fns-tz' const TIME_SERVER_URL_UTC = 'https://timeapi.io/api/Time/current/zone?timeZone=Etc/UTC' const MAX_ACCEPTABLE_RTT_MS = 1000 // Maximum acceptable round-trip time @@ -28,7 +29,9 @@ export const syncTimeOffset = createAsyncThunk('timeSync/syncTimeOffset', async let rtt = end - start // Calculate round-trip time console.debug('Time sync round trip time (unused):', rtt, 'ms') const data = await response.json() - const serverTime = new Date(data.dateTime + 'Z').getTime() + + // Convert server time (in the specified time zone) to UTC milliseconds + const serverTime = fromZonedTime(data.dateTime, data.timeZone).getTime() const currentTime = Date.now() return serverTime - currentTime diff --git a/webapp/src/icons/help/map_overview.png b/webapp/src/icons/help/map_overview.png new file mode 100644 index 000000000..ac6dcb007 Binary files /dev/null and b/webapp/src/icons/help/map_overview.png differ diff --git a/webapp/src/icons/help/organization_selection.png b/webapp/src/icons/help/organization_selection.png new file mode 100644 index 000000000..e34d6b145 Binary files /dev/null and b/webapp/src/icons/help/organization_selection.png differ diff --git a/webapp/src/icons/help/rsu_configure.png b/webapp/src/icons/help/rsu_configure.png new file mode 100644 index 000000000..f5b399231 Binary files /dev/null and b/webapp/src/icons/help/rsu_configure.png differ diff --git a/webapp/src/icons/help/rsu_count.png b/webapp/src/icons/help/rsu_count.png new file mode 100644 index 000000000..d06e49679 Binary files /dev/null and b/webapp/src/icons/help/rsu_count.png differ diff --git a/webapp/src/icons/help/rsu_count_menu.png b/webapp/src/icons/help/rsu_count_menu.png new file mode 100644 index 000000000..77d61cc5b Binary files /dev/null and b/webapp/src/icons/help/rsu_count_menu.png differ diff --git a/webapp/src/icons/help/rsu_popup_and_config_menu.png b/webapp/src/icons/help/rsu_popup_and_config_menu.png new file mode 100644 index 000000000..aabc309e9 Binary files /dev/null and b/webapp/src/icons/help/rsu_popup_and_config_menu.png differ diff --git a/webapp/src/icons/help/rsu_status_menu.png b/webapp/src/icons/help/rsu_status_menu.png new file mode 100644 index 000000000..b7fbb0088 Binary files /dev/null and b/webapp/src/icons/help/rsu_status_menu.png differ diff --git a/webapp/src/icons/rsu_configure.PNG b/webapp/src/icons/rsu_configure.PNG deleted file mode 100644 index 6404f32f1..000000000 Binary files a/webapp/src/icons/rsu_configure.PNG and /dev/null differ diff --git a/webapp/src/icons/rsu_count.PNG b/webapp/src/icons/rsu_count.PNG deleted file mode 100644 index f20c6c572..000000000 Binary files a/webapp/src/icons/rsu_count.PNG and /dev/null differ diff --git a/webapp/src/icons/rsu_heatmap.PNG b/webapp/src/icons/rsu_heatmap.PNG deleted file mode 100644 index 15780601e..000000000 Binary files a/webapp/src/icons/rsu_heatmap.PNG and /dev/null differ diff --git a/webapp/src/icons/rsu_menu.PNG b/webapp/src/icons/rsu_menu.PNG deleted file mode 100644 index 8e1590f0c..000000000 Binary files a/webapp/src/icons/rsu_menu.PNG and /dev/null differ diff --git a/webapp/src/icons/rsu_popup.PNG b/webapp/src/icons/rsu_popup.PNG deleted file mode 100644 index 0c6140e82..000000000 Binary files a/webapp/src/icons/rsu_popup.PNG and /dev/null differ diff --git a/webapp/src/icons/rsu_status.PNG b/webapp/src/icons/rsu_status.PNG deleted file mode 100644 index 1fe2482d7..000000000 Binary files a/webapp/src/icons/rsu_status.PNG and /dev/null differ diff --git a/webapp/src/models/MessageTypes.d.ts b/webapp/src/models/MessageTypes.d.ts index c756cb6b1..596ec6201 100644 --- a/webapp/src/models/MessageTypes.d.ts +++ b/webapp/src/models/MessageTypes.d.ts @@ -1,2 +1,2 @@ -export type MessageType = 'BSM' | 'PSM' | 'SSM' | 'SPAT' | 'SRM' | 'MAP' +export type MessageType = 'BSM' | 'PSM' | 'SSM' | 'SPAT' | 'SRM' | 'MAP' | 'TIM' export type GeoMessageType = 'BSM' | 'PSM' diff --git a/webapp/src/models/RsuApi.d.ts b/webapp/src/models/RsuApi.d.ts index 710aedd88..d74781562 100644 --- a/webapp/src/models/RsuApi.d.ts +++ b/webapp/src/models/RsuApi.d.ts @@ -37,10 +37,14 @@ export type RsuOnlineStatusRespSingle = { last_online: string | undefined } +export type MessageType = 'SPAT' | 'MAP' | 'BSM' | 'SRM' | 'SSM' | 'TIM' | 'PSM' + export type RsuCounts = { [ip: string]: { road: string - count: number + messageTypeCounts: { + [messageType: string]: number + } } } diff --git a/webapp/src/pages/Admin.tsx b/webapp/src/pages/Admin.tsx index a0f729f5c..ca5ea554c 100644 --- a/webapp/src/pages/Admin.tsx +++ b/webapp/src/pages/Admin.tsx @@ -1,5 +1,6 @@ import React, { useEffect } from 'react' -import { useDispatch } from 'react-redux' +import {useDispatch, useSelector} from 'react-redux' +import { selectOrganizationName } from '../generalSlices/userSlice' import { updateTableData as updateRsuTableData } from '../features/adminRsuTab/adminRsuTabSlice' import { updateTableData as updateIntersectionTableData } from '../features/adminIntersectionTab/adminIntersectionTabSlice' import { getAvailableUsers } from '../features/adminUserTab/adminUserTabSlice' @@ -19,13 +20,17 @@ import { evaluateFeatureFlags } from '../feature-flags' function Admin() { const dispatch: ThunkDispatch = useDispatch() + const organization = useSelector(selectOrganizationName) useEffect(() => { + // This preloads data for the admin pages + // Preload it with changes in dispatch and organization since it needs to be updated every time the organization is switched + // in order to show only RSUs, Intersections, and Users of selected organization if (evaluateFeatureFlags('rsu')) dispatch(updateRsuTableData()) if (evaluateFeatureFlags('intersection')) dispatch(updateIntersectionTableData()) dispatch(getAvailableUsers()) dispatch(getUserNotifications()) - }, [dispatch]) + }, [dispatch, organization]) return ( <> diff --git a/webapp/src/pages/IntersectionDashboard.tsx b/webapp/src/pages/IntersectionDashboard.tsx index 976ce752c..1873a0e85 100644 --- a/webapp/src/pages/IntersectionDashboard.tsx +++ b/webapp/src/pages/IntersectionDashboard.tsx @@ -1,7 +1,5 @@ -import React, { useEffect, useState } from 'react' +import React, { useState } from 'react' import { useDispatch, useSelector } from 'react-redux' -import { updateTableData as updateRsuTableData } from '../features/adminRsuTab/adminRsuTabSlice' -import { getAvailableUsers } from '../features/adminUserTab/adminUserTabSlice' import './css/NoTableWidth.css' import { NotFound } from './404' @@ -41,11 +39,6 @@ function IntersectionDashboard() { const [openMapDialog, setOpenMapDialog] = useState(false) const theme = useTheme() - useEffect(() => { - dispatch(updateRsuTableData()) - dispatch(getAvailableUsers()) - }, [dispatch]) - return ( <>
diff --git a/webapp/src/pages/Map.test.tsx b/webapp/src/pages/Map.test.tsx index 1e3f34ba2..ba591338b 100644 --- a/webapp/src/pages/Map.test.tsx +++ b/webapp/src/pages/Map.test.tsx @@ -72,6 +72,7 @@ it('snapshot bsmData clicked', () => { bsmCoordinates: [], rsuCounts: {}, mapList: [], + rsuData: [], bsmStart: '2023-05-10T03:24:00', bsmFilterStep: 60, // 1 hour bsmFilterOffset: 24 * 4, // 4 days diff --git a/webapp/src/pages/Map.tsx b/webapp/src/pages/Map.tsx index 9b07dc0a2..fe9cdbf3d 100644 --- a/webapp/src/pages/Map.tsx +++ b/webapp/src/pages/Map.tsx @@ -1,5 +1,5 @@ import React, { useEffect, useState, useMemo } from 'react' -import mapboxgl, { CircleLayer, FillLayer, LineLayer } from 'mapbox-gl' // This is a dependency of react-map-gl even if you didn't explicitly install it +import { CircleLayer, FillLayer, LineLayer } from 'mapbox-gl' // This is a dependency of react-map-gl even if you didn't explicitly install it import Map, { Marker, Popup, Source, Layer } from 'react-map-gl' import { Container } from 'reactstrap' import RsuMarker from '../components/RsuMarker' @@ -7,18 +7,15 @@ import EnvironmentVars from '../EnvironmentVars' import dayjs from 'dayjs' import { LocalizationProvider } from '@mui/x-date-pickers/LocalizationProvider' import { AdapterDayjs } from '@mui/x-date-pickers/AdapterDayjs' - +import TuneIcon from '@mui/icons-material/Tune' import { DateTimePicker } from '@mui/x-date-pickers/DateTimePicker' import Slider from '@mui/material/Slider' import { selectRsuOnlineStatus, selectRsuData, - selectRsuCounts, selectIssScmsStatusData, selectSelectedRsu, - selectMsgType, selectRsuIpv4, - selectHeatMapData, selectAddGeoMsgPoint, selectGeoMsgStart, selectGeoMsgEnd, @@ -70,7 +67,6 @@ import { } from '../generalSlices/configSlice' import ClearIcon from '@mui/icons-material/Clear' import ExpandMoreIcon from '@mui/icons-material/ExpandMore' -import ExpandLessIcon from '@mui/icons-material/ExpandLess' import { Button, FormGroup, @@ -98,7 +94,7 @@ import { Grid2, Stack, } from '@mui/material' - +import * as turf from '@turf/turf' import './css/MsgMap.css' import './css/Map.css' import { WZDxFeature, WZDxWorkZoneFeed } from '../models/wzdx/WzdxWorkZoneFeed42' @@ -112,8 +108,23 @@ import { useDispatch, useSelector } from 'react-redux' import { AnyAction, ThunkDispatch } from '@reduxjs/toolkit' import { RootState } from '../store' import { headerTabHeight } from '../styles/index' -import { selectActiveLayers, selectViewState, setMapViewState, toggleLayerActive } from './mapSlice' -import { selectMenuSelection, toggleMapMenuSelection } from '../features/menu/menuSlice' +import { + getClusterColorStops, + getClusterLabelSizeStops, + getClusterRadiusStops, + getHeatmapCountsStops, + selectActiveLayers, + selectViewState, + setMapViewState, + toggleLayerActive, +} from './mapSlice' +import { + selectCountsEndDate, + selectCountsMsgType, + selectCountsStartDate, + selectMenuSelection, + toggleMapMenuSelection, +} from '../features/menu/menuSlice' import { MapLayer } from '../models/MapLayer' import { toast } from 'react-hot-toast' import { RoomOutlined } from '@mui/icons-material' @@ -125,10 +136,10 @@ import { Feature, Point } from 'geojson' import { PrimaryButton } from '../styles/components/PrimaryButton' import { ConditionalRenderRsu, evaluateFeatureFlags } from '../feature-flags' import { DateTime } from 'luxon' - -// eslint-disable-next-line -// eslint-disable-next-line import/no-webpack-loader-syntax, @typescript-eslint/no-require-imports -;(mapboxgl as any).workerClass = require('worker-loader!mapbox-gl/dist/mapbox-gl-csp-worker').default +import { MessageType } from '../models/MessageTypes' +import { useGetRsuCountsQuery } from '../features/api/rsuCountsApiSlice' +import { CustomTable } from '../features/intersections/map/custom-table' +import { getStackGroupsByAxisId } from 'recharts/types/util/ChartUtils' const MILLISECONDS_PER_MINUTE = 60000 @@ -146,9 +157,7 @@ function MapPage() { const mapRef = React.useRef(null) const organization = useSelector(selectOrganizationName) const rsuData = useSelector(selectRsuData) - const rsuCounts = useSelector(selectRsuCounts) const selectedRsu = useSelector(selectSelectedRsu) - const countsMsgType = useSelector(selectMsgType) const issScmsStatusData = useSelector(selectIssScmsStatusData) const rsuOnlineStatus = useSelector(selectRsuOnlineStatus) const rsuIpv4 = useSelector(selectRsuIpv4) @@ -156,7 +165,9 @@ function MapPage() { const configCoordinates = useSelector(selectConfigCoordinates) const geoMsgType = useSelector(selectGeoMsgType) - const heatMapData = useSelector(selectHeatMapData) + const countsMsgType = useSelector(selectCountsMsgType) + const countsStartDate = useSelector(selectCountsStartDate) + const countsEndDate = useSelector(selectCountsEndDate) const geoMsgData = useSelector(selectGeoMsgData) const geoMsgCoordinates = useSelector(selectGeoMsgCoordinates) @@ -188,9 +199,10 @@ function MapPage() { const activeLayers = useSelector(selectActiveLayers) // RSU layer local state variables - const [selectedRsuCount, setSelectedRsuCount] = useState(null) const [displayType, setDisplayType] = useState('online') + const { data: rsuCounts } = useGetRsuCountsQuery({ organization, startDate: countsStartDate, endDate: countsEndDate }) + // Add these new state variables near the other source states const [previewPoint, setPreviewPoint] = useState | null>(null) @@ -510,6 +522,41 @@ function MapPage() { return availability }, [geoMsgData, startGeoMsgDate, filterStep, geoMsgFilterMaxOffset]) + const heatMapData = useMemo(() => { + return { + type: 'FeatureCollection' as 'FeatureCollection', + features: + rsuData + ?.map( + (rsu) => + ({ + type: 'Feature', + geometry: { + type: 'Point', + coordinates: [rsu.geometry.coordinates[0], rsu.geometry.coordinates[1]], + }, + properties: { + ipv4_address: rsu.properties.ipv4_address, + count: rsuCounts?.[rsu.properties.ipv4_address]?.messageTypeCounts?.[countsMsgType] ?? 0, + }, + } as GeoJSON.Feature) + ) + ?.filter((feature) => feature.properties.count > 0) ?? [], + } + }, [rsuData, rsuCounts, countsMsgType]) + + const rsuDataWithCounts = useMemo(() => { + return ( + rsuData?.map((rsu) => ({ + ...rsu, + properties: { + ...rsu.properties, + counts: rsuCounts?.[rsu.properties.ipv4_address]?.messageTypeCounts ?? {}, + }, + })) ?? [] + ) + }, [rsuData, rsuCounts]) + function dateChanged(e: Date, type: 'start' | 'end') { try { const date = DateTime.fromISO(e.toISOString()) @@ -571,6 +618,22 @@ function MapPage() { if (selectedWZDxMarkerIndex !== null) setSelectedWZDxMarker(wzdxMarkers[selectedWZDxMarkerIndex]) }, [selectedWZDxMarkerIndex, wzdxMarkers]) + const heatmapStops = useMemo(() => { + return getHeatmapCountsStops(countsMsgType, heatMapData) + }, [countsMsgType, heatMapData]) + + const clusterColorStops = useMemo(() => { + return getClusterColorStops(countsMsgType, heatMapData) + }, [countsMsgType, heatMapData]) + + const clusterRadiusStops = useMemo(() => { + return getClusterRadiusStops(countsMsgType, heatMapData) + }, [countsMsgType, heatMapData]) + + const clusterLabelSizeStops = useMemo(() => { + return getClusterLabelSizeStops(countsMsgType, heatMapData) + }, [countsMsgType, heatMapData]) + useEffect(() => { function createPopupTable(data: Array>) { const rows = [] @@ -698,18 +761,6 @@ function MapPage() { setSelectedWZDxMarkerIndex(null) } - function getStops() { - // populate tmp array with rsuCounts to get max count value - const max = Math.max(...Object.entries(rsuCounts).map(([, value]) => (value as { count: number }).count)) - const stopsArray = [[0, 0.25]] - let weight = 0.5 - for (let i = 1; i < max; i += 500) { - stopsArray.push([i, weight]) - weight += 0.25 - } - return stopsArray - } - const isOnline = () => { return rsuIpv4 in rsuOnlineStatus && Object.prototype.hasOwnProperty.call(rsuOnlineStatus[rsuIpv4], 'last_online') ? rsuOnlineStatus[rsuIpv4].last_online @@ -745,26 +796,27 @@ function MapPage() { setExpandedLayers((prev) => (prev.includes(layerId) ? prev.filter((id) => id !== layerId) : [...prev, layerId])) } - const layers: MapLayer[] = [ - { + const MAP_LAYERS: Record = { + RSU: { id: 'rsu-layer', label: 'RSU Viewer', type: 'symbol', tag: 'rsu', }, - { + HEATMAP: { id: 'heatmap-layer', label: 'Heatmap', type: 'heatmap', maxzoom: 14, source: 'heatMapData', + filter: ['all', ['has', 'count'], ['>', ['get', 'count'], 0]], paint: { 'heatmap-weight': { property: 'count', type: 'exponential', - stops: getStops(), + stops: heatmapStops, }, - 'heatmap-intensity': ['interpolate', ['linear'], ['zoom'], 0, 0, 10, 1, 13, 2], + 'heatmap-intensity': ['interpolate', ['linear'], ['zoom'], 0, 0, 9, 1, 10, 2], 'heatmap-color': [ 'interpolate', ['linear'], @@ -786,13 +838,19 @@ function MapPage() { }, tag: 'rsu', }, - { + HEATMAP_CLUSTER: { + id: 'heatmap-cluster', + label: 'Heatmap Cluster', + type: 'circle', + tag: 'rsu', + }, + MSG_VIEWER: { id: 'msg-viewer-layer', - label: 'V2x Message Viewer', + label: 'V2X Message Viewer', type: 'symbol', tag: 'rsu', }, - { + WZDX: { id: 'wzdx-layer', label: 'WZDx Viewer', type: 'line', @@ -802,25 +860,25 @@ function MapPage() { 'line-width': 8, }, }, - { + INTERSECTION: { id: 'intersection-layer', label: 'Intersections', type: 'symbol', tag: 'intersection', }, - { + MOOVE_AI: { id: 'moove-ai-layer', label: 'Moove AI Viewer', type: 'line', tag: 'mooveai', }, - { + HAAS_ALERT: { id: 'haas-alert-layer', label: 'HAAS Alert Viewer', type: 'circle', tag: 'haas', }, - ] + } const Legend = () => { const toggleLayer = (id: string) => { @@ -830,7 +888,6 @@ function MapPage() { case 'rsu-layer': dispatch(selectRsu(null)) dispatch(clearFirmware()) - setSelectedRsuCount(null) break case 'wzdx-layer': setSelectedWZDxMarkerIndex(null) @@ -848,6 +905,12 @@ function MapPage() { case 'wzdx-layer': dispatch(getWzdxData()) break + case 'heatmap-layer': + case 'heatmap-cluster': + if (!menuSelection.includes('Display Message Counts')) { + dispatch(toggleMapMenuSelection('Display Message Counts')) + } + break case 'moove-ai-layer': if (activeLayers.includes('msg-viewer-layer')) dispatch(toggleLayerActive('msg-viewer-layer')) break @@ -859,28 +922,17 @@ function MapPage() { return ( - {layers + {Object.values(MAP_LAYERS) .filter((layer) => evaluateFeatureFlags(layer.tag)) .map((layer) => (
- {layer.control && ( - toggleExpandLayer(layer.id)} - size="small" - edge="end" - aria-label={expandedLayers.includes(layer.id) ? 'Collapse' : 'Expand'} - > - {expandedLayers.includes(layer.id) ? : } - - )} toggleLayer(layer.id)} label={{layer.label}} control={} />
- {layer.control && {layer.control}}
))} @@ -894,7 +946,6 @@ function MapPage() { // Deselect any selected RSU when toggling point select tools dispatch(selectRsu(null)) dispatch(clearFirmware()) - setSelectedRsuCount(null) // Toggle the corresponding point select tool based on the origin if (origin === 'config') { @@ -1210,19 +1261,19 @@ function MapPage() { {activeLayers.includes('rsu-layer') && (
{configCoordinates.length >= 1 ? ( - + ) : null} {addConfigPoint && ( - + )}
)} - {rsuData?.map( + {rsuDataWithCounts?.map( (rsu) => activeLayers.includes('rsu-layer') && (selectedVendor === 'Select Vendor' || rsu['properties']['manufacturer_name'] === selectedVendor) && [ @@ -1240,9 +1291,6 @@ function MapPage() { dispatch(clearFirmware()) // TODO: Should remove?? dispatch(getRsuLastOnline(rsu.properties.ipv4_address)) dispatch(getIssScmsStatus()) - if (Object.prototype.hasOwnProperty.call(rsuCounts, rsu.properties.ipv4_address)) - setSelectedRsuCount(rsuCounts[rsu.properties.ipv4_address].count) - else setSelectedRsuCount(0) }} >
+
+
+
diff --git a/webapp/src/pages/css/Map.css b/webapp/src/pages/css/Map.css index e385f721c..a40fbcfa3 100644 --- a/webapp/src/pages/css/Map.css +++ b/webapp/src/pages/css/Map.css @@ -15,6 +15,11 @@ margin-right: 10px; } +/* Remove vertical padding from rsu popup */ +.rsu-popup .mapboxgl-popup-content { + padding: 0 !important; +} + .rw-dropdown-list-value { font-family: 'museo-slab', Arial, Helvetica, sans-serif; } diff --git a/webapp/src/pages/mapSlice.tsx b/webapp/src/pages/mapSlice.tsx index 0004e1686..a67ce87b5 100644 --- a/webapp/src/pages/mapSlice.tsx +++ b/webapp/src/pages/mapSlice.tsx @@ -2,6 +2,12 @@ import { createSlice } from '@reduxjs/toolkit' import EnvironmentVars from '../EnvironmentVars' import { RootState } from '../store' import { evaluateFeatureFlags } from '../feature-flags' +import { MessageType } from '../models/MessageTypes' + +const MESSAGE_COUNT_FREQUENCIES = { + SPAT: 10 * 86400, // 10 Hz for 24 hours + MAP: 86400, // 1 Hz for 24 hours +} as const const initialState = { mapViewState: EnvironmentVars.getMapboxInitViewState(), @@ -10,6 +16,132 @@ const initialState = { .map((layer) => layer.id), } +export function getClusterColorStops(msgType: MessageType, heatMapData: GeoJSON.FeatureCollection) { + // Safely calculate max with fallback + const counts = heatMapData.features + .map((f) => f.properties?.count as number) + .filter((count) => typeof count === 'number' && !isNaN(count) && count > 0) + + const max = counts.length > 0 ? Math.max(...counts) : 86400 // default to 1Hz for 24 hours + + // Get expected frequency for message type, fallback to actual max + const desiredValue = MESSAGE_COUNT_FREQUENCIES[msgType] ?? max + + // Prevent division by zero + if (desiredValue === 0) { + return [ + [0, '#ffffcc'], + [1, '#800026'], + ] as [number, string][] + } + + // Generate color stops based on message type frequency + return [ + [0, '#ffffcc'], // Light yellow (low) + [Math.round(desiredValue * 0.1), '#fed976'], + [Math.round(desiredValue * 0.2), '#feb24c'], + [Math.round(desiredValue * 0.3), '#fd8d3c'], + [Math.round(desiredValue * 0.4), '#fc4e2a'], // Orange-red (high) + [Math.round(desiredValue * 0.6), '#e31a1c'], + [Math.round(desiredValue * 0.8), '#bd0026'], // Deep red (very high) + [Math.round(desiredValue), '#800026'], // Dark red (extreme) + ] as [number, string][] +} + +export function getClusterRadiusStops(msgType: MessageType, heatMapData: GeoJSON.FeatureCollection) { + const counts = heatMapData.features + .map((f) => f.properties?.count as number) + .filter((count) => typeof count === 'number' && !isNaN(count) && count > 0) + + const max = counts.length > 0 ? Math.max(...counts) : 86400 + + const desiredValue = MESSAGE_COUNT_FREQUENCIES[msgType] ?? max + + if (desiredValue === 0) { + return [ + [0, 10], + [1, 20], + ] as [number, number][] + } + + // Generate radius stops based on message type frequency + return [ + [0, 0], // Min count = 0px radius + [1, 20], // 1 = 20px + [Math.round(desiredValue * 0.2), 22], // 20% = 22px + [Math.round(desiredValue * 0.4), 25], // 40% = 25px + [Math.round(desiredValue * 0.8), 30], // 80% = 30px + [Math.round(desiredValue), 35], // 100% = 35px + ] as [number, number][] +} + +export function getClusterLabelSizeStops( + msgType: MessageType, + heatMapData: GeoJSON.FeatureCollection +) { + // Safely calculate max with fallback + const counts = heatMapData.features + .map((f) => f.properties?.count as number) + .filter((count) => typeof count === 'number' && !isNaN(count) && count > 0) + + const max = counts.length > 0 ? Math.max(...counts) : 86400 // default to 1Hz for 24 hours + + // Get expected frequency for message type, fallback to actual max + const desiredValue = MESSAGE_COUNT_FREQUENCIES[msgType] ?? max + + // Prevent division by zero + if (desiredValue === 0) { + return [ + [0, 10], + [1, 16], + ] as [number, number][] + } + + // Generate text size stops based on message type frequency + // Text sizes range from 10px (low counts) to 16px (high counts) + return [ + [0, 10], // Min count = 10px font + [Math.round(desiredValue * 0.2), 11], // 20% = 11px + [Math.round(desiredValue * 0.4), 12], // 40% = 12px + [Math.round(desiredValue * 0.6), 13], // 60% = 13px + [Math.round(desiredValue * 0.8), 14], // 80% = 14px + [Math.round(desiredValue), 16], // 100% = 16px + ] as [number, number][] +} + +export function getHeatmapCountsStops(msgType: MessageType, heatMapData: GeoJSON.FeatureCollection) { + // Safely calculate max with fallback + const counts = heatMapData.features + .map((f) => f.properties?.count as number) + .filter((count) => typeof count === 'number' && !isNaN(count)) + + const max = counts.length > 0 ? Math.max(...counts) : 86400 // default to 1Hz for 24 hours + + // Get expected frequency for message type, fallback to actual max + const desiredValue = MESSAGE_COUNT_FREQUENCIES[msgType] ?? max + + // Prevent division by zero + if (desiredValue === 0) { + return [ + [0, 0], + [1, 1], + ] + } + + // Generate mapbox heatmap layer stops from 0 -> 1 + return [ + [0, 0], + [1, 0.1], // 1% = 0.1 intensity + [desiredValue * 0.05, 0.2], // 5% = 0.2 intensity + [desiredValue * 0.1, 0.3], // 10% = 0.3 intensity + [desiredValue * 0.2, 0.4], // 20% = 0.4 intensity + [desiredValue * 0.4, 0.6], // 40% = 0.6 intensity + [desiredValue * 0.6, 0.8], // 60% = 0.8 intensity + [desiredValue * 0.8, 0.9], // 80% = 0.9 intensity + [desiredValue, 1], // 100% = 1.0 intensity + ] as [number, number][] +} + export const mapSlice = createSlice({ name: 'map', initialState: { diff --git a/webapp/src/store.tsx b/webapp/src/store.tsx index 5d40d94d2..1e9cb8ae4 100644 --- a/webapp/src/store.tsx +++ b/webapp/src/store.tsx @@ -29,6 +29,7 @@ import intersectionMapReducer from './features/intersections/map/map-slice' import intersectionMapLayerStyleReducer from './features/intersections/map/map-layer-style-slice' import dataSelectorReducer from './features/intersections/data-selector/dataSelectorSlice' import { intersectionApiSlice } from './features/api/intersectionApiSlice' +import { rsuCountsApiSlice } from './features/api/rsuCountsApiSlice' import mapSliceReducer from './pages/mapSlice' import timeSyncReducer from './generalSlices/timeSyncSlice' import haasSliceReducer from './generalSlices/haasAlertSlice' @@ -69,6 +70,7 @@ export const setupStore = (preloadedState?: Partial) => { timeSync: timeSyncReducer, haas: haasSliceReducer, [intersectionApiSlice.reducerPath]: intersectionApiSlice.reducer, + [rsuCountsApiSlice.reducerPath]: rsuCountsApiSlice.reducer, }, preloadedState, middleware: (getDefaultMiddleware) => @@ -76,7 +78,9 @@ export const setupStore = (preloadedState?: Partial) => { thunk: true, serializableCheck: false, immutableCheck: false, - }).concat(intersectionApiSlice.middleware), + }) + .concat(intersectionApiSlice.middleware) + .concat(rsuCountsApiSlice.middleware), devTools: true, }) }