diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index 21bd9586..00000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve -title: '' -labels: bug -assignees: Sohambutala - ---- - -### Description -A clear and concise description of what the bug is. - -### Expected Behavior -A clear and concise description of what you expected to happen. - -### Actual Behavior -A clear and concise description of what is happening. - -**Environment (please complete the following information):** - - OS: [e.g. iOS, Windows] - - Mode [e.g. Prefect Cloud, Cloud VM, Ephemeral] - - Version [e.g. 0.1.4] - -### Steps to Reproduce -Steps to reproduce the behavior: -1. Go to '...' -2. Click on '....' -3. Scroll down to '....' -4. See error - -**Screenshots** -If applicable, add screenshots to help explain your problem. - -### Possible Solution / Suggestion -If applicable, add possible solution or suggestion to solve the bug - -### Additional Context -Include any additional context, logs, or screenshots that might help in diagnosing the issue. diff --git a/.github/workflows/ISSUE_TEMPLATE/bug_report.md b/.github/workflows/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index 11c90672..00000000 --- a/.github/workflows/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,29 +0,0 @@ -### Description -A clear and concise description of what the bug is. - -### Expected Behavior -A clear and concise description of what you expected to happen. - -### Actual Behavior -A clear and concise description of what is happening. - -**Environment (please complete the following information):** - - OS: [e.g. iOS, Windows] - - Mode [e.g. Prefect Cloud, Cloud VM, Ephemeral] - - Version [e.g. 0.1.4] - -### Steps to Reproduce -Steps to reproduce the behavior: -1. Go to '...' -2. Click on '....' -3. Scroll down to '....' -4. See error - -**Screenshots** -If applicable, add screenshots to help explain your problem. - -### Possible Solution / Suggestion -If applicable, add possible solution or suggestion to solve the bug - -### Additional Context -Include any additional context, logs, or screenshots that might help in diagnosing the issue. \ No newline at end of file diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml deleted file mode 100644 index 2f384ba3..00000000 --- a/.github/workflows/docker-build.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Build Docker Image - -on: - push: - branches: - - '*' - pull_request: - branches: - - '*' - -jobs: - - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Build the Docker image - run: docker build . --file Dockerfile --tag my-image-name:$(date +%s) - diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml deleted file mode 100644 index 863f552b..00000000 --- a/.github/workflows/docker-release.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Release Docker Image -on: - release: - types: [published] - -jobs: - - build-and-push: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - # Build the Docker image - - name: Build the Docker image - run: docker build . --file Dockerfile --tag my-image-name:${{ github.sha }} - - # Log in to Docker Hub (you need to set up DOCKER_USERNAME and DOCKER_PASSWORD in GitHub Secrets) - - name: Log in to Docker Hub - run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin - - # Push the Docker image to Docker Hub - - name: Push the Docker image - run: docker push my-image-name:${{ github.sha }} diff --git a/.github/workflows/ci.yaml b/.github/workflows/pr.yaml similarity index 51% rename from .github/workflows/ci.yaml rename to .github/workflows/pr.yaml index b1721a1f..64edf995 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/pr.yaml @@ -1,4 +1,4 @@ -name: CI +name: PR on: pull_request: @@ -6,30 +6,38 @@ on: branches: - main +permissions: + contents: read + jobs: tests: - name: Python 3.12 tests - runs-on: ubuntu-24.04 + name: Python ${{ matrix.python-version }} tests + runs-on: ubuntu-26.04 + strategy: + fail-fast: false + matrix: + python-version: ["3.13", "3.14", "3.15"] steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v7.0.1 with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v7.0.0 with: - python-version: "3.12" + python-version: ${{ matrix.python-version }} + allow-prereleases: true cache: pip - name: Install package and test dependencies run: | python -m pip install --upgrade pip - pip install -e ".[test,mission]" + python -m pip install -e ".[test,mission]" - name: Run tests run: | - pytest -vv \ + python -m pytest -vv \ --ignore=tests/test_operations_predict_hake.py \ --ignore=tests/test_postprocessing_flow_validation.py diff --git a/.github/workflows/pypi.yaml b/.github/workflows/pypi.yaml new file mode 100644 index 00000000..345fe48c --- /dev/null +++ b/.github/workflows/pypi.yaml @@ -0,0 +1,107 @@ +# Publish distributions to PyPI and TestPyPI using GitHub Actions. +name: Publish to PyPI + +on: + release: + types: [released] + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +jobs: + build-artifact: + name: Build echodataflow package + runs-on: ubuntu-24.04 + if: github.repository == 'echostack-org/echodataflow' || github.event_name == 'workflow_dispatch' + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + # setuptools-scm needs the release tags and Git history. + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v7.0.0 + with: + python-version: '3.12' + + - name: Install build frontend + run: python -m pip install --upgrade pip build twine + + # pyproject.toml already uses no-local-version for index-compatible versions. + - name: Build and validate source and wheel distributions + run: | + python -m build + python -m twine check --strict dist/* + ls -lh dist/ + + - name: Upload distributions + uses: actions/upload-artifact@v7 + with: + name: releases + path: dist/ + if-no-files-found: error + + test-built-dist: + name: Test echodataflow package + needs: build-artifact + runs-on: ubuntu-24.04 + steps: + - name: Set up Python + uses: actions/setup-python@v7.0.0 + with: + python-version: '3.12' + + - name: Download distributions + uses: actions/download-artifact@v8 + with: + name: releases + path: dist/ + + - name: Publish to TestPyPI + uses: pypa/gh-action-pypi-publish@v1.14.2 + with: + password: ${{ secrets.TEST_PYPI_API_TOKEN }} + repository-url: https://test.pypi.org/legacy/ + verbose: true + skip-existing: true + + # Test these exact artifacts rather than another version from an index. + - name: Check wheel installation + run: | + python -m pip install --upgrade pip + python -m pip install dist/*.whl + python -m pip check + python -c "import echodataflow; print(echodataflow.__version__)" + echodataflow-deploy --help + + - name: Check source distribution installation + run: | + python -m pip uninstall --yes echodataflow + python -m pip install dist/*.tar.gz + python -m pip check + python -c "import echodataflow; print(echodataflow.__version__)" + echodataflow-deploy --help + + publish-pypi: + name: Publish echodataflow to PyPI + needs: test-built-dist + if: >- + github.repository == 'echostack-org/echodataflow' && + github.event_name == 'release' && + startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-24.04 + steps: + - name: Download distributions + uses: actions/download-artifact@v8 + with: + name: releases + path: dist/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@v1.14.2 + with: + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml deleted file mode 100644 index 07f12c33..00000000 --- a/.github/workflows/pypi.yml +++ /dev/null @@ -1,40 +0,0 @@ -# This workflow will upload a Python Package using Twine when a release is created -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries - -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -name: echodataflow-pypi-publish - -on: - release: - types: [published] - -permissions: - contents: read - -jobs: - deploy: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5.1.0 - with: - python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install build - - name: Build package - run: python -m build - - name: Publish package - uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} - diff --git a/docs/source/_toc.yml b/docs/source/_toc.yml index f2266ea5..17af24c6 100644 --- a/docs/source/_toc.yml +++ b/docs/source/_toc.yml @@ -10,7 +10,6 @@ parts: sections: - file: configuration/datastore.md - file: configuration/pipeline.md - - file: gettingstarteddocker - caption: Demo chapters: - file: local/intro @@ -34,7 +33,6 @@ parts: sections: - file: logging/daskstream.md - file: logging/awscloud - - file: logging/kelk - caption: Command-Line Interface chapters: - file: cli/intro @@ -44,4 +42,4 @@ parts: - file: cli/credentials - caption: Appendix chapters: - - file: configuration/blocks.md \ No newline at end of file + - file: configuration/blocks.md diff --git a/docs/source/gettingstarteddocker.md b/docs/source/gettingstarteddocker.md deleted file mode 100644 index 43d2f57b..00000000 --- a/docs/source/gettingstarteddocker.md +++ /dev/null @@ -1,42 +0,0 @@ -# Docker - -Using Docker to manage and deploy Echodataflow ensures a consistent and reproducible environment, simplifying the setup and configuration process. - -This guide will walk you through the steps to pull the Echodataflow Docker image from Docker Hub, run it with the necessary network settings, and configure it to interact with Prefect for workflow orchestration. - -## Prerequisites -Before you begin, ensure that you have the following installed on your host machine: - -- [`Docker`](https://www.docker.com/get-started/): The platform for developing, shipping, and running applications in containers. - -## Pulling the Docker Image -To download the latest version of the Echodataflow Docker image from Docker Hub, use the following command: - -```bash -docker pull blackdranzer/echodataflow -``` - -## Running the Docker Image -To run the Echodataflow image with network settings configured to interact with Prefect, use this command: - -```bash -docker run --network="host" -e PREFECT_API_URL=http://host.docker.internal:4200/api blackdranzer/echodataflow -``` - -### Important Note on Configuration -The `PREFECT_API_URL` environment variable is essential for connecting the Docker container to a Prefect dashboard, enabling the monitoring and triggering of flows externally. This URL can be configured in two primary ways: - -- Local Prefect Server: - -Start a local server using the command: - -```bash -prefect server start -``` - -Ensure that Prefect is installed on your host machine, which can be done following these [installation instructions](https://docs.prefect.io/latest/getting-started/installation/). - -- Prefect Cloud: Utilize a Cloud API URL if you prefer cloud services. Be mindful of the [API rate limits](https://docs.prefect.io/latest/cloud/rate-limits/) associated with your Prefect Cloud account, which vary by plan. - -## Additional Resources -For more detailed guidance on deploying Prefect with Docker, refer to the [Prefect Docker Deployment Guide](https://docs.prefect.io/latest/guides/docker/#building-a-docker-image). \ No newline at end of file diff --git a/docs/source/logging/kelk.md b/docs/source/logging/kelk.md deleted file mode 100644 index 966aaec7..00000000 --- a/docs/source/logging/kelk.md +++ /dev/null @@ -1,89 +0,0 @@ -# Implementing Logging with Kafka and Elastic Stack - -## Overview -Echodataflow integrates with Kafka to send worker messages to a Kafka topic, which can then be routed to any logging destination. In this example, we will set up an [Elastic Stack](https://www.elastic.co/) (Elasticsearch, Logstash, Kibana, Beats) with Kafka integration to analyze and visualize logs in real-time. - -### Setting Up the Infrastructure -Dockerized Elastic Stack with Kafka Integration - -This setup includes: - -- `Elasticsearch` for powerful search and data analytics -- `Logstash` for log processing and ingestion into Elasticsearch -- `Kibana` for data visualization with dashboards and analytics -- `Apache Kafka` with Zookeeper for reliable log streaming -- `Schema Registry` for managing Kafka data schemas -- `Confluent Control Center` for Kafka cluster management and monitoring -- `Metricbeat` for monitoring system metrics and Docker containers - -**Prerequisites** -Ensure you have Docker and Docker Compose installed on your machine. A basic understanding of Docker, the ELK Stack, and Apache Kafka is recommended. - -### Quick Start - -1. Clone the Repository - -```bash -git clone https://github.com/Sohambutala/StreamLogging.git -cd StreamLogging -``` - -2. Start the Services - -```bash -docker-compose up -d -# This command starts all the services defined in the docker-compose.yml file in detached mode. -``` - -3. Access the Services - -- `Kibana`: Open http://localhost:5601 to access Kibana's web interface. -- `Confluent Control Center`: Visit http://localhost:9021 to manage and monitor your Kafka cluster. -- `Elasticsearch`: Accessible at http://localhost:9200. - -4. Monitor with Metricbeat - -Metricbeat is configured to collect metrics from Docker containers and the host system. Check the Metricbeat dashboards in Kibana for insights. - -### Configuration -You can customize the configurations for each service by modifying their respective configuration files in the repository: - -- `Elasticsearch`: ./elasticsearch/elasticsearch.yml -- `Logstash`: ./logstash/config/logstash.yml and the pipeline files in ./logstash/pipeline/ -- `Kibana`: ./kibana/kibana.yml -- `Metricbeat`: ./metricbeat/metricbeat.yml - -### Stopping the Services -To stop all services and remove containers, networks, and volumes created by docker-compose up, run: - -```bash -docker-compose down -v -``` - -## Configuring Kafka Logging in Echodataflow -Configure your Kafka server in the datastore.yaml file for echodataflow. - -### Example datastore.yaml - -```yaml -name: Bell_M._Shimada-SH1707-EK60 -sonar_model: EK60 -raw_regex: (.*)-?D(?P\w{1,8})-T(?P