diff --git a/.github/workflows/artifact-publish.yml b/.github/workflows/artifact-publish.yml index c3e8a0b966..f94d16d213 100644 --- a/.github/workflows/artifact-publish.yml +++ b/.github/workflows/artifact-publish.yml @@ -7,6 +7,9 @@ on: jobs: build: runs-on: ubuntu-latest + permissions: + contents: read + packages: write steps: - name: Checkout code (with submodules) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 56e0942338..fe511176fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,28 +7,28 @@ on: - "develop" - "master" - "release/*" - jobs: docker: runs-on: ubuntu-latest steps: - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v7 with: cache-from: type=gha cache-to: type=gha,mode=max test-no-sonar: - if: github.event.pull_request.base.repo.owner.login != 'usdot-jpo-ode' + # Fallback for PRs from forks outside usdot-jpo-ode (e.g. cdot). Runs plain tests without SonarCloud since SONAR_TOKEN is not available to fork-triggered workflows. + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.owner.login != 'usdot-jpo-ode' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: submodules: recursive - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: "21" distribution: "temurin" @@ -36,14 +36,18 @@ jobs: run: mvn clean verify sonar: - if: github.event.pull_request.base.repo.owner.login == 'usdot-jpo-ode' + # Run SonarCloud analysis on: + # - pushes to develop/master/release branches (internal) + # - PRs opened from branches within the usdot-jpo-ode org + # Skipped for fork PRs (e.g. from cdot) because GitHub does not expose SONAR_TOKEN to workflows triggered from forks. Those PRs fall through to the test-no-sonar job instead. + if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.owner.login == 'usdot-jpo-ode') runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: submodules: recursive - name: Set up JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: "21" distribution: "temurin" @@ -51,5 +55,4 @@ jobs: env: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} run: | - ls -la && pwd - mvn -e clean org.jacoco:jacoco-maven-plugin:prepare-agent package sonar:sonar -Dsonar.projectKey=usdot.jpo.ode:jpo-ode -Dsonar.projectName=jpo-ode -Dsonar.organization=usdot-jpo-ode -Dsonar.host.url=https://sonarcloud.io -Dsonar.branch.name=$GITHUB_REF_NAME + mvn -e clean verify sonar:sonar -Dsonar.projectKey=usdot.jpo.ode:jpo-ode -Dsonar.projectName=jpo-ode -Dsonar.organization=usdot-jpo-ode -Dsonar.host.url=https://sonarcloud.io diff --git a/.sonarqube/sonar-scanner.properties b/.sonarqube/sonar-scanner.properties deleted file mode 100644 index 6ba40a7ded..0000000000 --- a/.sonarqube/sonar-scanner.properties +++ /dev/null @@ -1,45 +0,0 @@ -sonar.modules= asn1_codec, \ - jpo-cvdp, \ - jpo-ode-common, \ - jpo-ode-consumer-example, \ - jpo-ode-core, \ - jpo-ode-plugins, \ - jpo-ode-svcs, \ - jpo-sdw-depositor, \ - jpo-security-svcs, \ - jpo-asn-runtime, \ - jpo-asn-j2735-2024 - -asn1_codec.sonar.projectBaseDir = /home/circleci/project/asn1_codec -jpo-cvdp.sonar.projectBaseDir = /home/circleci/project/jpo-cvdp -jpo-ode-common.sonar.projectBaseDir = /home/circleci/project/jpo-ode-common -jpo-ode-consumer-example.sonar.projectBaseDir = /home/circleci/project/jpo-ode-consumer-example -jpo-ode-core.sonar.projectBaseDir = /home/circleci/project/jpo-ode-core -jpo-ode-plugins.sonar.projectBaseDir = /home/circleci/project/jpo-ode-plugins -jpo-ode-svcs.sonar.projectBaseDir = /home/circleci/project/jpo-ode-svcs -jpo-sdw-depositor.sonar.projectBaseDir = /home/circleci/project/jpo-sdw-depositor -jpo-security-svcs.sonar.projectBaseDir = /home/circleci/project/jpo-security-svcs -jpo-asn-runtime.sonar.projectBaseDir = /home/circleci/project/jpo-asn-pojos/jpo-asn-runtime -jpo-asn-j2735-2024.sonar.projectBaseDir = /home/circleci/project/jpo-asn-pojos/jpo-asn-j2735-2024 - -asn1_codec.sonar.sources = src -jpo-cvdp.sonar.sources = src -jpo-ode-common.sonar.sources = src -jpo-ode-consumer-example.sonar.sources = src -jpo-ode-core.sonar.sources = src -jpo-ode-plugins.sonar.sources = src -jpo-ode-svcs.sonar.sources = src -jpo-sdw-depositor.sonar.sources = src -jpo-security-svcs.sonar.sources = src -jpo-asn-runtime.sonar.sources = src -jpo-asn-j2735-2024.sonar.sources = src - -jpo-ode-common.sonar.tests = src/test -jpo-ode-consumer-example.sonar.tests = src/test -jpo-ode-core.sonar.tests = src/test -jpo-ode-plugins.sonar.tests = src/test -jpo-ode-svcs.sonar.tests = src/test -jpo-sdw-depositor.sonar.tests = src/test -jpo-security-svcs.sonar.tests = src/test -jpo-asn-runtime.sonar.tests = src/test -jpo-asn-j2735-2024.sonar.tests = src/test \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000..9b5e4019df --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/README.md b/README.md index b7579deb03..ba074aac92 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,4 @@ -| Sonar Code Quality | Sonar Code Coverage | -|---------------------|---------------------| -[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=usdot.jpo.ode%3Ajpo-ode&metric=alert_status)](https://sonarcloud.io/dashboard?id=usdot.jpo.ode%3Ajpo-ode) | [![Coverage](https://sonarcloud.io/api/project_badges/measure?project=usdot.jpo.ode%3Ajpo-ode&metric=coverage)](https://sonarcloud.io/dashboard?id=usdot.jpo.ode%3Ajpo-ode) | +![GitHub Release](https://img.shields.io/github/v/release/usdot-jpo-ode/jpo-ode) [![CI](https://github.com/usdot-jpo-ode/jpo-ode/actions/workflows/ci.yml/badge.svg)](https://github.com/usdot-jpo-ode/jpo-ode/actions/workflows/ci.yml) ![Docker Pulls](https://img.shields.io/docker/pulls/usdotjpoode/jpo-ode) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=usdot.jpo.ode%3Ajpo-ode&metric=alert_status)](https://sonarcloud.io/dashboard?id=usdot.jpo.ode%3Ajpo-ode) ![GitHub License](https://img.shields.io/github/license/usdot-jpo-ode/jpo-ode) # jpo-ode @@ -12,6 +10,8 @@ The ITS ODE is a real-time virtual data router that ingests and processes operat _Figure 1: ODE Dataflows_ +For a module-by-module description of the components in this diagram, see [Architecture Section 5.2 – Architecture Module Reference](docs/Architecture.md#52---architecture-module-reference). + **Documentation:** 1. [ODE Architecture](docs/Architecture.md) @@ -444,6 +444,23 @@ In order to utilize Confluent Cloud: #### Note +### Port Mapped Receiver Configuration +Many production deployments may not send intersection data directly from an RSU unit. This often happens if another device is responsible for generating MAP / SPaT messages or if a proxy or load balancer is used between an RSU unit and the ODE. To handle these edge cases the ODE has a configurable port-mapped receiver which can be used for receiving data on a variety of ports. The intent of this feature is to allow the ODE to continue to identify different source RSU units based upon the port the data is forwarded on. + +To use port-mapped ingest, add an entry similar to the below to the application.yaml file. The entry below creates a BSM receiver which listens on port 1234. Any BSM data received on port 1234 will be marked as having been received from the IP 1.2.3.4. + +``` +port-mapped-ingest: + sources: + - intersectipon-name: "Name" + intersection-id: 1234 + origin-ip: 1.2.3.4 + port: 1234 + type: BSM +``` + +Please note, ports added here are not automatically opened in the host docker container. Any port added here should separately be added to the running docker container. + This has only been tested with Confluent Cloud but technically all SASL authenticated Kafka brokers can be reached using this method. [Back to top](#table-of-contents) @@ -464,6 +481,8 @@ The configuration that defines this is in the jpo-utils submodule [here](jpo-uti For further documentation on configuring the MongoDB Kafka Connect image refer [to this readme](jpo-utils/README.md). + + ## Note Kafka connect is being used for MongoDB in this implementation but it can interact with many types of databases, here is further documentation for [kafka connect](https://docs.confluent.io/platform/current/connect/index.html) diff --git a/asn1_codec b/asn1_codec index 1b6dc4c0ea..a51fc0a8c1 160000 --- a/asn1_codec +++ b/asn1_codec @@ -1 +1 @@ -Subproject commit 1b6dc4c0eac3a741ad88ddff4fae3a005091b408 +Subproject commit a51fc0a8c1ced0654b61667f9fcb79cf799e4db1 diff --git a/docker-compose.yml b/docker-compose.yml index 9633120e57..1da3587939 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -54,6 +54,7 @@ services: KAFKA_KEY_SERIALIZER: ${KAFKA_KEY_SERIALIZER} KAFKA_VALUE_SERIALIZER: ${KAFKA_VALUE_SERIALIZER} KAFKA_PARTITIONER_CLASS: ${KAFKA_PARTITIONER_CLASS} + KAFKA_AUTO_COMMIT_INTERVAL: ${KAFKA_AUTO_COMMIT_INTERVAL} ODE_TIM_INGEST_MONITORING_ENABLED: ${ODE_TIM_INGEST_MONITORING_ENABLED} ODE_TIM_INGEST_MONITORING_INTERVAL: ${ODE_TIM_INGEST_MONITORING_INTERVAL} ODE_STOMP_EXPORTER_ENABLED: ${ODE_STOMP_EXPORTER_ENABLED} @@ -63,6 +64,7 @@ services: ODE_ROCKSDB_BLOCK_SIZE: ${ODE_ROCKSDB_BLOCK_SIZE} ODE_ROCKSDB_N_MEMTABLES: ${ODE_ROCKSDB_N_MEMTABLES} ODE_ROCKSDB_MEMTABLE_SIZE: ${ODE_ROCKSDB_MEMTABLE_SIZE} + ODE_KAFKA_LISTENER_CONCURRENCY: ${ODE_KAFKA_LISTENER_CONCURRENCY} depends_on: kafka: condition: service_healthy @@ -103,6 +105,7 @@ services: ACM_LOG_TO_CONSOLE: ${ADM_LOG_TO_CONSOLE} ACM_LOG_TO_FILE: ${ADM_LOG_TO_FILE} ACM_LOG_LEVEL: ${ADM_LOG_LEVEL} + ACM_NUMBER_OF_PROCESSES: ${ADM_NUMBER_OF_PROCESSES:-1} depends_on: kafka: condition: service_healthy @@ -136,6 +139,7 @@ services: ACM_LOG_TO_CONSOLE: ${AEM_LOG_TO_CONSOLE} ACM_LOG_TO_FILE: ${AEM_LOG_TO_FILE} ACM_LOG_LEVEL: ${AEM_LOG_LEVEL} + ACM_NUMBER_OF_PROCESSES: ${AEM_NUMBER_OF_PROCESSES:-1} depends_on: kafka: condition: service_healthy diff --git a/docs/Architecture.md b/docs/Architecture.md index d51dc89855..f49d8d5fc3 100644 --- a/docs/Architecture.md +++ b/docs/Architecture.md @@ -17,20 +17,28 @@ _Last updated April 26th, 2024_ # Contents - [Version History](#version-history) -- [1 - Introduction](#introduction) -- [2 - Project Overview](#project-overview) -- [3 - System Overview](#system-overview) - - [3.1 - ODE Technology Stack](#ode-technology-stack) - - [3.2 - Producer Mechanisms](#producer-mechanisms) - - [3.3 - Consumer Mechanisms](#consumer-mechanisms) - - [3.4 - ODE Management Console](#ode-management-console) -- [4 - Architecture Pattern](#architecture-pattern) - - [4.1 - Pattern Description](#pattern-description) - - [4.2 - Pattern Topology](#pattern-topology) -- [5 - JPO ODE Micro-services Topology](#jpo-ode-micro-services-topology) - - [5.1 - Deployments](#deployments) -- [6 - Appendix](#appendix) - - [6.1 - Glossary](#glossary) +- [1 - Introduction](#1---introduction) +- [2 - Project Overview](#2---project-overview) +- [3 - System Overview](#3---system-overview) + - [3.1 - ODE Technology Stack](#31---ode-technology-stack) + - [3.2 - Producer Mechanisms](#32---producer-mechanisms) + - [3.3 - Consumer Mechanisms](#33---consumer-mechanisms) + - [3.4 - ODE Management Console](#34---ode-management-console) +- [4 - Architecture Pattern](#4---architecture-pattern) + - [4.1 - Pattern Description](#41---pattern-description) + - [4.2 - Pattern Topology](#42---pattern-topology) +- [5 - JPO ODE Micro-services Topology](#5---jpo-ode-micro-services-topology) + - [5.1 - Deployments](#51---deployments) + - [5.2 - Architecture Module Reference](#52---architecture-module-reference) + - [Core ODE Services](#core-ode-services) + - [Data Import Services](#data-import-services) + - [Data Processing Services](#data-processing-services) + - [Data Export Services](#data-export-services) + - [System-Wide Services](#system-wide-services) + - [Supporting Libraries](#supporting-libraries) + - [Notes](#notes) +- [6 - Appendix](#6---appendix) + - [6.1 - Glossary](#61---glossary) @@ -352,7 +360,7 @@ _Figure 7 - JPO ODE Micro-services Topology_ -### 5.1 - Deployments +## 5.1 - Deployments Docker is utilized as the primary deployment mechanism to compartmentalize each of the designed micro-services into separate @@ -361,6 +369,76 @@ containers each running a distinct service. The ODE application runs in one container, its submodules run in separate containers and other major frameworks such as Kafka run in their own separate containers. + + +## 5.2 - Architecture Module Reference + +This section provides implementation details for the modules shown in the +ODE TMC architecture diagrams, including feature descriptions, interfaces, +deployment roles, and source code references. + +### Core ODE Services + +| Module | Purpose | Interface(s) | Implementation / Codebase | +|--------|---------|--------------|----------------------------| +| REST API | Accepts Traveler Information Messages (TIM), Probe Data Messages (PDM), and management requests into the ODE processing pipeline. | HTTP/REST | [jpo-ode](https://github.com/usdot-jpo-ode/jpo-ode) | +| Kafka Data Streams | Primary internal messaging bus used for inter-service communication and publish/subscribe routing of JSON and POJO message streams. | Kafka Producer/Consumer APIs | [jpo-ode](https://github.com/usdot-jpo-ode/jpo-ode), [jpo-utils](https://github.com/usdot-jpo-ode/jpo-utils) | +| UDP (ASN.1) Input | Receives ASN.1 UPER encoded J2735 messages for direct ingestion and decoding. | UDP | [jpo-ode](https://github.com/usdot-jpo-ode/jpo-ode) | +| Log Upload Folder | File-based ingestion mechanism for uploaded OBU and infrastructure log files. | File System / SCP | [jpo-ode](https://github.com/usdot-jpo-ode/jpo-ode) | +| SNMP to RSU | Supports TIM query, deposit, and deletion operations with roadside units (RSUs). | SNMP / NTCIP1218 / RSU MIB 4.1 | [jpo-ode](https://github.com/usdot-jpo-ode/jpo-ode) | + +### Data Import Services + +| Module | Purpose | Interface(s) | Implementation / Codebase | +|--------|---------|--------------|----------------------------| +| BSM Importer | Ingests and routes Basic Safety Messages (BSM). | Kafka, UDP, File Upload | [jpo-ode](https://github.com/usdot-jpo-ode/jpo-ode) | +| TIM Importer | Ingests and routes Traveler Information Messages (TIM). | REST, Kafka, SNMP | [jpo-ode](https://github.com/usdot-jpo-ode/jpo-ode) | +| Distress Notification Importer | Imports distress notification events for downstream processing. | Kafka / REST | [jpo-ode](https://github.com/usdot-jpo-ode/jpo-ode) | + +### Data Processing Services + +| Module | Purpose | Interface(s) | Implementation / Codebase | +|--------|---------|--------------|----------------------------| +| Privacy Protection Module | Filters and sanitizes BSM data to reduce privacy risk and support geofencing/privacy policies. | Kafka | [jpo-cvdp](https://github.com/usdot-jpo-ode/jpo-cvdp) | +| ASN.1 Message Deduplication Module | Removes duplicate ASN.1 messages before downstream processing/archiving. | Kafka | [jpo-deduplicator](https://github.com/usdot-jpo-ode/jpo-deduplicator) | +| ASN.1 Decoder Module | Decodes ASN.1 UPER/COER encoded J2735 messages into XER. | Kafka | [asn1_codec](https://github.com/usdot-jpo-ode/asn1_codec) | +| ASN.1 Encoder Module | Encodes XER messages into ASN.1 formats for outbound transmission. | Kafka | [asn1_codec](https://github.com/usdot-jpo-ode/asn1_codec) | +| HAAS Alert J2735 Module | Processes HAAS alerts messaging and generates J2735 messages for V2X transmission. | Kafka | [jpo-haas-asn1-bridge](https://github.com/usdot-jpo-ode/jpo-haas-asn1-bridge) | +| RTK Ingest J2735 Module | Imports and processes RTK correction messages. | Kafka / UDP | [jpo-rtk-asn1-bridge](https://github.com/usdot-jpo-ode/jpo-rtk-asn1-bridge) | +| GeoJSON Converter | Converts location-based data into GeoJSON format. | Internal service | [jpo-geojsonconverter](https://github.com/usdot-jpo-ode/jpo-geojsonconverter) | +| Conflict Monitor | Supports conflict detection and monitoring workflows. | Kafka | [jpo-conflictmonitor](https://github.com/usdot-jpo-ode/jpo-conflictmonitor) | + +### Data Export Services + +| Module | Purpose | Interface(s) | Implementation / Codebase | +|--------|---------|--------------|----------------------------| +| S3 Depositor Module | Stores exported ODE data into AWS S3-compatible storage. | AWS S3 API | [jpo-ode](https://github.com/usdot-jpo-ode/jpo-ode) | +| JPO MEC MQTT Depositor | Publishes processed data to external MQTT brokers or MEC environments. | MQTT | [jpo-mec-deposit](https://github.com/usdot-jpo-ode/jpo-mec-deposit) | +| SNMP Depositor | Sends TIM and related messages to RSUs via SNMP. | SNMP | [jpo-ode](https://github.com/usdot-jpo-ode/jpo-ode) | +| SDX Depositor | Publishes data to Situational Data Exchange (SDX). | External API | [jpo-sdw-depositor](https://github.com/usdot-jpo-ode/jpo-sdw-depositor) | + +### System-Wide Services + +| Module | Purpose | Interface(s) | Implementation / Codebase | +|--------|---------|--------------|----------------------------| +| Kafka Message Bus | Distributed event backbone for ODE services. | Kafka | [jpo-utils](https://github.com/usdot-jpo-ode/jpo-utils) | +| Security Module | Supports IEEE 1609.2 signing, encryption, and certificate management. | REST / Internal service | [jpo-security-svcs](https://github.com/usdot-jpo-ode/jpo-security-svcs) | +| Message Database (MongoDB) | Optional persistence layer for ODE output topics. | MongoDB / Kafka Connect | [jpo-utils](https://github.com/usdot-jpo-ode/jpo-utils) | + +### Supporting Libraries + +| Library | Purpose | Codebase | +|--------|---------|----------| +| ASN.1 POJOs | Java POJOs representing J2735 ASN.1 message schemas. | [jpo-asn-pojos](https://github.com/usdot-jpo-ode/jpo-asn-pojos) | +| ODE Output Validator Library | Schema validation and output verification utilities. | [ode-output-validator-library](https://github.com/usdot-jpo-ode/ode-output-validator-library) | + +### Notes + +- Optional modules may be enabled through Docker Compose profiles or + deployment-specific configuration. +- Repository links reflect the current public source locations for each + major service or library. + # 6 - Appendix diff --git a/docs/Release_notes.md b/docs/Release_notes.md index 6e9ad8d88c..1a635c6c78 100644 --- a/docs/Release_notes.md +++ b/docs/Release_notes.md @@ -1,6 +1,19 @@ JPO-ODE Release Notes ---------------------------- +Version 6.0.0, released May 2026 +---------------------------------------- +### **Summary** +This release expands support for existing jpo-ode functionality and improves flexibility in deployment and data handling. JSON schemas are now bundled within the jpo-asn-pojos submodule, enabling offline usage without external dependencies. Additionally, a receiving port remapping configuration has been introduced to better identify traffic sources that may not originate from an RSU. This release also includes several minor fixes. + +Enhancements in this release: +- [ASN Hex 1609.2 Signature Retention (ITSA)](https://github.com/usdot-jpo-ode/jpo-ode/pull/596) +- [Submodule update to provide JSON schemas in resources for offline availability](https://github.com/neaeraconsulting/jpo-ode/pull/1) +- [Port remap receiver](https://github.com/neaeraconsulting/jpo-ode/pull/2) +- [Configurable kafka concurrency](https://github.com/neaeraconsulting/jpo-ode/pull/3) +- [Docker shared Kafka fix](https://github.com/neaeraconsulting/jpo-ode/pull/9) +- [Add env vars for number of ACM processes, update test process](https://github.com/neaeraconsulting/jpo-ode/pull/7) + Version 5.1.0, released October 2025 ---------------------------------------- ### **Summary** diff --git a/docs/compatibility.md b/docs/compatibility.md index 98f6598a94..3b00a98684 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -3,6 +3,7 @@ This table serves as a guide, suggesting which versions of individual submodules | [ODE (this project)](https://github.com/usdot-jpo-ode/jpo-ode/releases) | [ACM](https://github.com/usdot-jpo-ode/asn1_codec/releases) | [PPM](https://github.com/usdot-jpo-ode/jpo-cvdp/releases) | [SEC](https://github.com/usdot-jpo-ode/jpo-security-svcs/releases) | [SDWD](https://github.com/usdot-jpo-ode/jpo-sdw-depositor/releases) | [S3D](https://github.com/usdot-jpo-ode/jpo-s3-deposit/releases) | [GJConverter](https://github.com/usdot-jpo-ode/jpo-geojsonconverter/releases) | [CMonitor](https://github.com/usdot-jpo-ode/jpo-conflictmonitor/releases) | [CVisualizer](https://github.com/usdot-jpo-ode/jpo-conflictvisualizer/releases) | [CVManager](https://github.com/usdot-jpo-ode/jpo-cvmanager/releases) | [MEC](https://github.com/usdot-jpo-ode/jpo-mec-deposit/releases) | |-------|-------|-------|-------|-------|-------|-------|-------|-------|-------|-------| +| 6.0.0 | 3.3.0 | 1.6.0 | 1.7.0 | 1.10.0 | 1.7.1 | 3.3.0 | 3.2.0 | N/A | 2.1.0 | 1.1.0 | | 5.1.0 | 3.2.0 | 1.6.0 | 1.7.0 | 1.10.0 | 1.7.1 | 3.2.0 | 3.1.0 | N/A | 2.0.0 | 1.0.0 | | 4.1.2 | 3.1.0 | 1.5.0 | 1.6.0 | 1.9.1 | 1.7.1 | 2.1.0 | 2.1.0 | 1.5.0 | 1.6.0 | N/A | | 4.0.0 | 3.0.0 | 1.5.0 | 1.5.0 | 1.9.0 | 1.7.0 | 2.0.0 | 2.0.0 | 1.5.0 | 1.5.0 | N/A | diff --git a/docs/images/readme/figure1.png b/docs/images/readme/figure1.png index ff5ca5d5b6..4e349f36ec 100644 Binary files a/docs/images/readme/figure1.png and b/docs/images/readme/figure1.png differ diff --git a/docs/release_process_quarterly.md b/docs/release_process_quarterly.md index 4d6263a7e5..8f55b2fcf6 100644 --- a/docs/release_process_quarterly.md +++ b/docs/release_process_quarterly.md @@ -43,6 +43,31 @@ None - [ ] messages get decoded as expected - [ ] messages get encoded as expected +#### Details on how to do tests + +Run compilation and unit tests in Docker: From the root directory of asn1_codec issue: +``` +docker build -t acm . +docker run --env-file .env --name acm -it acm +``` +In a separate shell: +``` +docker exec -it acm bash +/build/acm_tests +``` + +where the `.env` file is configured with necessary environment variables. + +To test program startup and encoding/decoding: + +Pull the release branch into the ODE submodule, and run the ODE via `docker compose up --build -d` from the jpo-ode root directory, with the following profiles set in the `.env` file: +``` +COMPOSE_PROFILES=ode,adm,aem,kafka,kafka_setup,kafka_ui +``` +To test decoding send test messages with the `udpsender_generic.py` script and verify the messages are decoded by looking at the `topic.Asn1DecoderOutput` in the kafka UI. + +To test encoding, use the Kafka UI to produce one of the decoded messages from `topic.Asn1DecoderOutput` to `topic.Asn1EncoderInput` and check the decoded output appears on `topic.Asn1EncoderOutput`. + ### 3. Project Reference Updates & Release Creation - [ ] Merge ‘release/(year)-(quarter)’ branch into ‘master’ branch for the asn1_codec project - [ ] Create a release for the asn1_codec project from the ‘master’ branch and tag the release with the version number of the release. (e.g. asn1_codec-x.x.x) @@ -220,11 +245,13 @@ None - [ ] code compiles - gradle build - [ ] `jpo-asn-j2735-2024` subproject builds via gradle - - [ ] `jpo-asn-test-generator` subproject build via gradle + - [ ] `jpo-asn-test-generator` subproject builds via gradle + - [ ] `jpo-asn-jsonschema-generator` subproject builds via gradle - maven build: - [ ] `jpo-asn-j2735-2024` subproject builds via maven - [ ] unit tests pass - [ ] test generator command line tool can generate messages + - [ ] schema generator command line tool can generate schemas ### 3. Project Reference Updates & Release Creation - [ ] Update Gradle build version number diff --git a/jpo-asn-pojos b/jpo-asn-pojos index c7d446df1b..1ff018d20a 160000 --- a/jpo-asn-pojos +++ b/jpo-asn-pojos @@ -1 +1 @@ -Subproject commit c7d446df1ba24d6bf7f485ce7c112423c9cf6f26 +Subproject commit 1ff018d20a64e7f0d8f57f039252618ab9e978b4 diff --git a/jpo-ode-common/pom.xml b/jpo-ode-common/pom.xml index c01fb219c9..d47f8bc29c 100644 --- a/jpo-ode-common/pom.xml +++ b/jpo-ode-common/pom.xml @@ -1,18 +1,20 @@ - + 4.0.0 usdot.jpo.ode jpo-ode - 5.1.0 + 6.0.0 - + jpo-ode-common jpo-ode-common JPO ODE Common Components jar - + src/main/java/us/dot/its/jpo/ode/util/GeoUtils.java @@ -21,26 +23,26 @@ src/main/java/us/dot/its/jpo/ode/inet/* - + org.json json - 20231013 + 20231013 com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat - jackson-dataformat-xml + com.fasterxml.jackson.dataformat + jackson-dataformat-xml - + jakarta.xml.bind jakarta.xml.bind-api 4.0.0 - + javax.websocket javax.websocket-client-api @@ -52,4 +54,4 @@ 4.0.0 - + \ No newline at end of file diff --git a/jpo-ode-core/pom.xml b/jpo-ode-core/pom.xml index c5ea2bfd1d..5aca2eddd8 100644 --- a/jpo-ode-core/pom.xml +++ b/jpo-ode-core/pom.xml @@ -7,7 +7,7 @@ usdot.jpo.ode jpo-ode - 5.1.0 + 6.0.0 jpo-ode-core @@ -25,12 +25,12 @@ usdot.jpo.ode jpo-ode-common - 5.1.0 + 6.0.0 usdot.jpo.ode jpo-ode-plugins - 5.1.0 + 6.0.0 org.apache.httpcomponents @@ -90,13 +90,13 @@ com.networknt json-schema-validator - 1.0.76 + 1.0.88 test usdot.jpo.asn jpo-asn-j2735-2024 - 1.1.0 + 1.2.0 diff --git a/jpo-ode-core/src/main/java/us/dot/its/jpo/ode/model/OdeLogMetadata.java b/jpo-ode-core/src/main/java/us/dot/its/jpo/ode/model/OdeLogMetadata.java index 85117710ff..9d540916c3 100644 --- a/jpo-ode-core/src/main/java/us/dot/its/jpo/ode/model/OdeLogMetadata.java +++ b/jpo-ode-core/src/main/java/us/dot/its/jpo/ode/model/OdeLogMetadata.java @@ -102,6 +102,14 @@ public OdeLogMetadata(OdeMsgPayload payload) { super(payload); } + /** + * Same as {@link #OdeLogMetadata(OdeMsgPayload)} but sets {@code asn1} to {@code asn1Hex} + * directly instead of from the payload's stripped bytes. + */ + public OdeLogMetadata(OdeMsgPayload payload, String asn1Hex) { + super(payload, asn1Hex); + } + public OdeLogMetadata() { super(); } diff --git a/jpo-ode-core/src/main/java/us/dot/its/jpo/ode/model/OdeMessageFrameMetadata.java b/jpo-ode-core/src/main/java/us/dot/its/jpo/ode/model/OdeMessageFrameMetadata.java index bf7750c059..e5b84184ca 100644 --- a/jpo-ode-core/src/main/java/us/dot/its/jpo/ode/model/OdeMessageFrameMetadata.java +++ b/jpo-ode-core/src/main/java/us/dot/its/jpo/ode/model/OdeMessageFrameMetadata.java @@ -36,6 +36,14 @@ public OdeMessageFrameMetadata(OdeMsgPayload payload) { super(payload); } + /** + * Same as {@link #OdeMessageFrameMetadata(OdeMsgPayload)} but sets {@code asn1} to the given hex + * string (for example the full UDP datagram payload before 1609.3 header stripping). + */ + public OdeMessageFrameMetadata(OdeMsgPayload payload, String asn1Hex) { + super(payload, asn1Hex); + } + public OdeMessageFrameMetadata(Source source) { this.source = source; } diff --git a/jpo-ode-core/src/main/java/us/dot/its/jpo/ode/model/OdeMsgMetadata.java b/jpo-ode-core/src/main/java/us/dot/its/jpo/ode/model/OdeMsgMetadata.java index 3091f33ee0..61d06b7f46 100644 --- a/jpo-ode-core/src/main/java/us/dot/its/jpo/ode/model/OdeMsgMetadata.java +++ b/jpo-ode-core/src/main/java/us/dot/its/jpo/ode/model/OdeMsgMetadata.java @@ -24,9 +24,9 @@ /** * Base ODE Metadata class. */ -@JsonPropertyOrder({ "logFileName", "recordType", "receivedMessageDetails", "payloadType", "serialId", - "odeReceivedAt", "schemaVersion", "maxDurationTime", "recordGeneratedAt", "recordGeneratedBy", "sanitized", - "asn1" }) +@JsonPropertyOrder({"logFileName", "recordType", "receivedMessageDetails", "payloadType", + "serialId", "odeReceivedAt", "schemaVersion", "maxDurationTime", "recordGeneratedAt", + "recordGeneratedBy", "sanitized", "asn1"}) public class OdeMsgMetadata extends OdeObject { public static final String METADATA_STRING = "metadata"; @@ -71,11 +71,24 @@ public OdeMsgMetadata(OdeMsgPayload payload) { } /** - * Constructs an OdeMsgMetadata object with the specified payload, serial ID, - * and received time. + * Same as {@link #OdeMsgMetadata(OdeMsgPayload)} but sets {@code asn1} to {@code asn1Hex} + * directly (for example the full received UDP hex) instead of from the payload's stripped + * {@code bytes} field. * - * @param payload the payload to be set - * @param serialId the serial ID to be set + * @param payload the payload (type, serial id, and receive time are initialized like the + * single-argument constructor) + * @param asn1Hex hex string for {@link #setAsn1(String)}; may be null + */ + public OdeMsgMetadata(OdeMsgPayload payload, String asn1Hex) { + this(payload.getClass().getName(), new SerialId(), DateTimeUtils.now()); + setAsn1(asn1Hex); + } + + /** + * Constructs an OdeMsgMetadata object with the specified payload, serial ID, and received time. + * + * @param payload the payload to be set + * @param serialId the serial ID to be set * @param receivedAt the time the message was received */ private OdeMsgMetadata(OdeMsgPayload payload, SerialId serialId, String receivedAt) { @@ -84,12 +97,12 @@ private OdeMsgMetadata(OdeMsgPayload payload, SerialId serialId, String received } /** - * Constructs an OdeMsgMetadata object with the specified payload type, serial - * ID, and received time. + * Constructs an OdeMsgMetadata object with the specified payload type, serial ID, and received + * time. * * @param payloadType the type of the payload - * @param serialId the serial ID to be set - * @param receivedAt the time the message was received + * @param serialId the serial ID to be set + * @param receivedAt the time the message was received */ public OdeMsgMetadata(String payloadType, SerialId serialId, String receivedAt) { super(); @@ -198,9 +211,11 @@ public void setAsn1(String asn1) { } /** - * Sets the ASN1 value for the metadata object. + * Sets {@code asn1} from the serialized {@code bytes} field of the payload (typically the + * header-stripped ASN.1 used for decoding). For full received hex (e.g. UDP), prefer + * {@link #OdeMsgMetadata(OdeMsgPayload, String)} or {@link #setAsn1(String)}. * - * @param payload the ASN1 payload hex string + * @param payload the payload whose bytes are copied into {@code asn1} */ public void setAsn1(OdeMsgPayload payload) { if (payload != null && payload.getData() != null) { diff --git a/jpo-ode-plugins/pom.xml b/jpo-ode-plugins/pom.xml index 6b1324f33f..8cc5642827 100644 --- a/jpo-ode-plugins/pom.xml +++ b/jpo-ode-plugins/pom.xml @@ -1,5 +1,7 @@ - + 4.0.0 jpo-ode-plugins @@ -11,7 +13,7 @@ usdot.jpo.ode jpo-ode - 5.1.0 + 6.0.0 @@ -27,7 +29,7 @@ usdot.jpo.ode jpo-ode-common - 5.1.0 + 6.0.0 usdot-jpo-ode @@ -75,7 +75,7 @@ pom - org.jmockit + com.github.hazendaz.jmockit jmockit ${jmockit.version} test @@ -151,7 +151,8 @@ maven-surefire-plugin 3.2.5 - -javaagent:${user.home}/.m2/repository/org/jmockit/jmockit/${jmockit.version}/jmockit-${jmockit.version}.jar + @{argLine} + -javaagent:${user.home}/.m2/repository/com/github/hazendaz/jmockit/jmockit/${jmockit.version}/jmockit-${jmockit.version}.jar -Xshare:off @@ -198,6 +199,28 @@ + + + org.jacoco + jacoco-maven-plugin + ${jacoco.version} + + + prepare-agent + + prepare-agent + + + + report + test + + report + + + + + diff --git a/sample.env b/sample.env index c46436951b..16a1f04417 100644 --- a/sample.env +++ b/sample.env @@ -63,6 +63,9 @@ DEFAULT_SNMP_PROTOCOL= ######################### # Kafka and Confluent Cloud Properties +# While not strictly required for the ODE to operate properly. Setting this value is required to allow applications outside of the ODE's docker network to connect to Kafka. +KAFKA_BOOTSTRAP_SERVERS=${DOCKER_HOST_IP}:9092 + # The type of Kafka broker to connect to. If set to "CONFLUENT", the broker will be Confluent Cloud. Otherwise, it will be a local Kafka broker. KAFKA_TYPE= KAFKA_LINGER_MS=1 @@ -78,6 +81,7 @@ KAFKA_RETRIES=0 KAFKA_KEY_SERIALIZER="org.apache.kafka.common.serialization.StringSerializer" KAFKA_VALUE_SERIALIZER="org.apache.kafka.common.serialization.StringSerializer" KAFKA_PARTITIONER_CLASS="org.apache.kafka.clients.producer.internals.DefaultPartitioner" +KAFKA_AUTO_COMMIT_INTERVAL=1000 # Confluent Cloud API access credentials (only required if KAFKA_TYPE is set to "CONFLUENT") CONFLUENT_KEY= @@ -141,4 +145,11 @@ ODE_TIM_INGEST_MONITORING_INTERVAL=60 # ODE STOMP Exporter for Demo UI # Set this to false to save resources when running the application in a production environement. The StompStringExporter # is exclusively used to support the Demo Console hosted at http://localhost:8080 -ODE_STOMP_EXPORTER_ENABLED=true \ No newline at end of file +ODE_STOMP_EXPORTER_ENABLED=true + +# Kafka listener concurrency +ODE_KAFKA_LISTENER_CONCURRENCY=1 + +# Number of processes for ACM +ADM_NUMBER_OF_PROCESSES=1 +AEM_NUMBER_OF_PROCESSES=1 \ No newline at end of file