ccollins/CUMULUS 4428 Add Input Granules

CHANGELOG.md

@@ -94,6 +94,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
   - Added pdr-cleanup task into cumulus core from ASDC
 - **CUMULUS-4563**
   - Added a Github action to generate requirements.txt files from coreified uv.lock files
+- **CUMULUS-4428**
+  - Added add-input-granules task
+- **CUMULUS-4426**
+  - Added unit tests for add-input-granules task
 
 ### Changed
 

tasks/add-input-granules/README.md

@@ -0,0 +1,67 @@
+# @cumulus/add-input-granules
+
+This task adds a list of granules processed by child ingest executions to its output message. It uses the [Cumulus Python API](https://github.com/nasa/Cumulus-API-Python) to fetch granule information.
+
+## Usage
+This task is intended to be used as part of PDR workflows. It is run after all ingest workflows are completed and prior to granule cleanup tasks to which it provides input granules.
+
+### Input
+
+| field name | type | default | required | values | description
+| ---------- | ---- | ------- | -------- | ------ | -----------
+| pdr | object | N/A | yes | N/A | Product Delivery Record
+| pdr.name | string | N/A | yes | N/A | PDR filename
+| pdr.path | string | N/A | yes | N/A | PDR location
+| pdr.archivePath | string | N/A | no | N/A | Archived PDR location
+| running | array[string] | N/A | yes | N/A | List of queued and running workflow execution ARNS
+| completed | array[string]  | N/A | yes | N/A | List of completed workflow execution ARNs
+| failed | array[object] | N/A | yes | N/A | List of failed workflow ARNs and reason for failure
+| failed[].arn | string | N/A | yes | N/A | Failed execution ARN
+| failed[].reason | string | N/A | yes | N/A | Reason for workflow failure
+
+### Output
+
+| field name | type | default | required | values | description
+| ---------- | ---- | ------- | -------- | ------ | -----------
+| pdr | object | N/A | yes | N/A | Product Delivery Record
+| pdr.name | string | N/A | yes | N/A | PDR filename
+| pdr.path | string | N/A | yes | N/A | PDR location
+| pdr.archivePath | string | N/A | yes | N/A | Archived PDR location
+| running | array[string] | N/A | yes | N/A | List of queued and running workflow execution ARNS
+| completed | array[string]  | N/A | yes | N/A | List of completed workflow execution ARNs
+| failed | array[object] | N/A | yes | N/A | List of failed workflow ARNs and reason for failure
+| failed[].arn | string | N/A | yes | N/A | Failed execution ARN
+| failed[].reason | string | N/A | yes | N/A | Reason for workflow failure
+| granules | array[object] | N/A | yes | N/A | List of granules
+| granules[].granuleId | string | N/A | yes | N/A | Granule ID
+| granules[].files | array[object] | N/A | yes | N/A | List of files associated with granule
+| granules[].files[].bucket | string | N/A | yes | N/A | Bucket where file is archived in S3
+| granules[].files[].checksum | string | N/A | no | N/A | Checksum value for file
+| granules[].files[].fileName | string | N/A | no | N/A | Name of file (e.g. file.txt)
+| granules[].files[].key | string | N/A | yes | N/A | S3 Key for archived file
+| granules[].files[].size | integer | N/A | no | N/A | Size of file (in bytes)
+| granules[].files[].source | string | N/A | no | N/A | Source URI of the file from origin system (e.g. S3, FTP, HTTP)
+| granules[].files[].type | string | N/A | no | N/A | Type of file (e.g. data, metadata, browse)
+
+### Example workflow configuration and use
+
+
+### Internal Dependencies
+
+This task uses the Cumulus Private API Lambda via the Cumulus Python API and requires the `PRIVATE_API_LAMBDA_ARN` environment variable to be set.
+
+### External Dependencies
+
+ - https://github.com/nasa/Cumulus-API-Python
+ - https://github.com/nasa/cumulus-message-adapter
+ - https://github.com/nasa/cumulus-message-adapter-python
+
+## Contributing
+
+To make a contribution, please [see our Cumulus contributing guidelines](https://github.com/nasa/cumulus/blob/master/CONTRIBUTING.md) and our documentation on [adding a task](https://nasa.github.io/cumulus/docs/adding-a-task)
+
+## About Cumulus
+
+Cumulus is a cloud-based data ingest, archive, distribution and management prototype for NASA's future Earth science data streams.
+
+[Cumulus Documentation](https://nasa.github.io/cumulus)

tasks/add-input-granules/deploy/main.tf

@@ -0,0 +1,20 @@
+module "add_input_granules_task" {
+  source = "../../../tf-modules/cumulus-task"
+
+  name               = "AddInputGranules"
+  prefix             = var.prefix
+  role               = var.lambda_processing_role_arn
+  lambda_zip_path    = "${path.module}/../dist/final/lambda.zip"
+  subnet_ids         = var.lambda_subnet_ids
+  security_group_id  = var.lambda_security_group_id
+  timeout            = var.lambda_timeout
+  memory_size        = var.lambda_memory_size
+  runtime            = "python3.12"
+  log_retention_days = var.log_retention_days
+  layers             = [var.cumulus_message_adapter_lambda_layer_version_arn]
+  environment = {
+    PRIVATE_API_LAMBDA_ARN = var.private_api_lambda_arn
+  }
+
+  tags = var.tags
+}

tasks/add-input-granules/deploy/outputs.tf

@@ -0,0 +1,9 @@
+output "add_input_granules_lambda" {
+  description = "The task lambda function"
+  value       = module.add_input_granules_task.cumulus_task_lambda
+}
+
+output "add_input_granules_log_group" {
+  description = "The task lambda function log group"
+  value       = module.add_input_granules_task.cumulus_task_log_group_name
+}

tasks/add-input-granules/deploy/variables.tf

@@ -0,0 +1,53 @@
+variable "cumulus_message_adapter_lambda_layer_version_arn" {
+  description = "ARN of the Cumulus Message Adapter Lambda layer"
+  type        = string
+}
+
+variable "lambda_memory_size" {
+  description = "Memory size for the Lambda function in MB"
+  type        = number
+  default     = 512
+}
+
+variable "lambda_processing_role_arn" {
+  description = "IAM ARN for the Lambda execution role"
+  type        = string
+}
+
+variable "lambda_security_group_id" {
+  description = "Security group ID for Lambda VPC configuration"
+  type        = string
+  default     = ""
+}
+
+variable "lambda_subnet_ids" {
+  description = "List of subnet IDs for Lambda VPC configuration"
+  type        = list(string)
+  default     = []
+}
+
+variable "log_retention_days" {
+  description = "The number of days to retain logs in CloudWatch"
+  type        = number
+}
+
+variable "lambda_timeout" {
+  description = "Timeout value for the Lambda function in seconds"
+  type        = number
+  default     = 300
+}
+
+variable "prefix" {
+  type        = string
+}
+
+variable "private_api_lambda_arn" {
+  description = "The ARN of the private API Lambda function"
+  type        = string
+}
+
+variable "tags" {
+  description = "Tags to be applied to managed resources"
+  type        = map(string)
+  default     = {}
+}

tasks/add-input-granules/deploy/versions.tf

@@ -0,0 +1,10 @@
+terraform {
+  required_version = ">= 1.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.0"
+    }
+  }
+}

tasks/add-input-granules/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "@cumulus/add-input-granules-task",
+  "private": true,
+  "version": "21.3.1",
+  "description": "Adds input granules to the output message using execution ARNs",
+  "main": "index.js",
+  "homepage": "https://github.com/nasa/cumulus/tree/master/tasks/add-input-granules",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/nasa/cumulus"
+  },
+  "scripts": {
+    "test": "uv run pytest -q",
+    "lint": "uv run ruff check . && uv run ruff format --diff . && uv run mypy src",
+    "clean": "rm -rf dist && rm -rf .venv && mkdir -p dist/packages",
+    "build": "uv sync",
+    "prepare": "npm run build",
+    "package": "npm run clean && cp -r ./schemas ./dist/packages && ../../bin/package_uv.sh 3.12 ${PWD}",
+    "install-python-deps": "npm run build"
+  },
+  "publishConfig": {
+    "access": "restricted"
+  },
+  "nyc": {
+    "exclude": [
+      "tests"
+    ]
+  },
+  "author": "Cumulus Authors",
+  "license": "Apache-2.0"
+}

tasks/add-input-granules/pyproject.toml

@@ -0,0 +1,30 @@
+[project]
+name = "add-input-granules"
+version = "21.3.1"
+description = "Adds input granules to the output message using execution ARNs"
+readme = "README.md"
+requires-python = "~=3.12.0"
+dependencies = [
+    "cumulus-api",
+    "cumulus-message-adapter-python>=2.4.0",
+]
+
+[dependency-groups]
+dev = [
+    "mypy~=1.19",
+    "pytest~=9.0",
+    "pytest-cov~=7.0",
+    "ruff~=0.14.0",
+]
+
+[tool.pytest]
+minversion = "9.0"
+pythonpath = [
+    "src",
+]
+testpaths = [
+    "tests",
+]
+
+[tool.uv.sources]
+cumulus-api = { git = "https://github.com/nasa/Cumulus-API-Python", tag = "v3.1.0" }