diff --git a/aws/plugin.go b/aws/plugin.go index 2d3a36c..217318d 100755 --- a/aws/plugin.go +++ b/aws/plugin.go @@ -16,6 +16,7 @@ import ( "github.com/turbot/tailpipe-plugin-aws/tables/s3_server_access_log" "github.com/turbot/tailpipe-plugin-aws/tables/vpc_flow_log" "github.com/turbot/tailpipe-plugin-aws/tables/waf_traffic_log" + "github.com/turbot/tailpipe-plugin-aws/tables/securityhub_finding" "github.com/turbot/tailpipe-plugin-sdk/plugin" "github.com/turbot/tailpipe-plugin-sdk/row_source" "github.com/turbot/tailpipe-plugin-sdk/table" @@ -38,6 +39,7 @@ func init() { table.RegisterTable[*guardduty_finding.GuardDutyFinding, *guardduty_finding.GuardDutyFindingTable]() table.RegisterTable[*nlb_access_log.NlbAccessLog, *nlb_access_log.NlbAccessLogTable]() table.RegisterTable[*s3_server_access_log.S3ServerAccessLog, *s3_server_access_log.S3ServerAccessLogTable]() + table.RegisterTable[*securityhub_finding.SecurityHubFinding, *securityhub_finding.SecurityHubFindingTable]() table.RegisterTable[*vpc_flow_log.VpcFlowLog, *vpc_flow_log.VpcFlowLogTable]() table.RegisterTable[*waf_traffic_log.WafTrafficLog, *waf_traffic_log.WafTrafficLogTable]() diff --git a/docs/sources/aws_s3_bucket.md b/docs/sources/aws_s3_bucket.md index 2d25877..0d1a4dc 100644 --- a/docs/sources/aws_s3_bucket.md +++ b/docs/sources/aws_s3_bucket.md @@ -77,6 +77,7 @@ The following tables define their own default values for certain source argument - **[aws_guardduty_finding](https://hub.tailpipe.io/plugins/turbot/aws/tables/aws_guardduty_finding#aws_s3_bucket)** - **[aws_nlb_access_log](https://hub.tailpipe.io/plugins/turbot/aws/tables/aws_nlb_access_log#aws_s3_bucket)** - **[aws_s3_server_access_log](https://hub.tailpipe.io/plugins/turbot/aws/tables/aws_s3_server_access_log#aws_s3_bucket)** +- **[aws_securityhub_finding](https://hub.tailpipe.io/plugins/turbot/aws/tables/aws_securityhub_finding#aws_s3_bucket)** - **[aws_cost_and_usage_focus](https://hub.tailpipe.io/plugins/turbot/aws/tables/aws_cost_and_usage_focus#aws_s3_bucket)** - **[aws_cost_and_usage_report](https://hub.tailpipe.io/plugins/turbot/aws/tables/aws_cost_and_usage_report#aws_s3_bucket)** - **[aws_cost_optimization_recommendation](https://hub.tailpipe.io/plugins/turbot/aws/tables/aws_cost_optimization_recommendation#aws_s3_bucket)** diff --git a/docs/tables/aws_securityhub_finding/index.md b/docs/tables/aws_securityhub_finding/index.md new file mode 100644 index 0000000..4669818 --- /dev/null +++ b/docs/tables/aws_securityhub_finding/index.md @@ -0,0 +1,224 @@ +--- +title: "Tailpipe Table: aws_securityhub_finding - Query AWS Security Hub Findings" +description: "AWS Security Hub findings provide comprehensive security findings from various AWS security services and partner integrations, including details about potential security issues and compliance violations." +--- + +# Table: aws_securityhub_finding - Query AWS Security Hub Findings + +The `aws_securityhub_finding` table allows you to query data from [AWS Security Hub findings](https://docs.aws.amazon.com/securityhub/latest/userguide/securityhub-findings.html). This table provides detailed information about potential security issues and compliance violations detected across your AWS accounts and resources, including severity levels, compliance status, affected resources, and recommended remediation steps. + +## Configure + +Create a [partition](https://tailpipe.io/docs/manage/partition) for `aws_securityhub_finding` ([examples](https://hub.tailpipe.io/plugins/turbot/aws/tables/aws_securityhub_finding#example-configurations)): + +```sh +vi ~/.tailpipe/config/aws.tpc +``` + +```hcl +connection "aws" "security_account" { + profile = "my-security-account" +} + +partition "aws_securityhub_finding" "my_findings" { + source "aws_s3_bucket" { + connection = connection.aws.security_account + bucket = "aws-securityhub-findings-bucket" + } +} +``` + +## Collect + +[Collect](https://tailpipe.io/docs/manage/collection) findings for all `aws_securityhub_finding` partitions: + +```sh +tailpipe collect aws_securityhub_finding +``` + +Or for a single partition: + +```sh +tailpipe collect aws_securityhub_finding.my_findings +``` + +## Query + +**[Explore example queries for this table →](https://hub.tailpipe.io/plugins/turbot/aws/queries/aws_securityhub_finding)** + +### High Severity Findings + +List all high severity security findings with detailed resource information. + +```sql +select + tp_timestamp, + title, + types, + severity, + description, + tp_index as account_id, + region, + resources, + remediation.recommendation.text as remediation_text +from + aws_securityhub_finding +where + severity.normalized >= 70 +order by + severity.normalized desc, + tp_timestamp desc; +``` + +### Findings by Type + +Group findings by type with severity and temporal information. + +```sql +select + types, + count(*) as finding_count, + round(avg(severity.normalized), 2) as avg_severity +from + aws_securityhub_finding +group by + types +order by + finding_count desc; +``` + +### Recent Findings with Resource Details + +Examine recent security findings with comprehensive resource and remediation information. + +```sql +select + tp_timestamp, + title, + types, + severity, + resources, + tp_index as account_id, + region, + workflow_state, + remediation.recommendation.text as remediation_text +from + aws_securityhub_finding +where + created_at > current_date - interval '7 days' +order by + tp_timestamp desc; +``` + +## Example Configurations + +### Collect findings from an S3 bucket + +Collect Security Hub findings stored in an S3 bucket using the default log file format. + +```hcl +connection "aws" "security_account" { + profile = "my-security-account" +} + +partition "aws_securityhub_finding" "my_findings" { + source "aws_s3_bucket" { + connection = connection.aws.security_account + bucket = "aws-securityhub-findings-bucket" + } +} +``` + +### Collect findings from an S3 bucket with a prefix + +Collect Security Hub findings stored in an S3 bucket using a prefix. + +```hcl +partition "aws_securityhub_finding" "my_findings_prefix" { + source "aws_s3_bucket" { + connection = connection.aws.security_account + bucket = "aws-securityhub-findings-bucket" + prefix = "my/prefix/" + } +} +``` + +### Collect findings from local files + +You can also collect Security Hub findings from local files. + +```hcl +partition "aws_securityhub_finding" "local_findings" { + source "file" { + paths = ["/Users/myuser/securityhub_findings"] + file_layout = `%{DATA}.jsonl.gz` + } +} +``` + +### Filter high severity findings only + +Use the filter argument in your partition to focus on high severity findings, reducing the size of local storage. + +```hcl +partition "aws_securityhub_finding" "high_severity_findings" { + filter = "severity.normalized >= 70" + + source "aws_s3_bucket" { + connection = connection.aws.security_account + bucket = "aws-securityhub-findings-bucket" + } +} +``` + +### Collect findings for all accounts in an organization + +For a specific organization, collect findings for all accounts and regions. + +```hcl +partition "aws_securityhub_finding" "my_findings_org" { + source "aws_s3_bucket" { + connection = connection.aws.security_account + bucket = "securityhub-findings-bucket" + file_layout = `AWSLogs/o-aa111bb222/%{NUMBER:account_id}/SecurityHub/%{DATA:region}/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{DATA}.jsonl.gz` + } +} +``` + +### Collect findings for a single account + +For a specific account, collect findings for all regions. + +```hcl +partition "aws_securityhub_finding" "my_findings_account" { + source "aws_s3_bucket" { + connection = connection.aws.security_account + bucket = "securityhub-findings-bucket" + file_layout = `AWSLogs/(%{DATA:org_id}/)?123456789012/SecurityHub/%{DATA:region}/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{DATA}.jsonl.gz` + } +} +``` + +### Collect findings for a single region + +For all accounts, collect findings from us-east-1. + +```hcl +partition "aws_securityhub_finding" "my_findings_region" { + source "aws_s3_bucket" { + connection = connection.aws.security_account + bucket = "securityhub-findings-bucket" + file_layout = `AWSLogs/(%{DATA:org_id}/)?%{NUMBER:account_id}/SecurityHub/us-east-1/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{DATA}.jsonl.gz` + } +} +``` + +## Source Defaults + +### aws_s3_bucket + +This table sets the following defaults for the [aws_s3_bucket source](https://hub.tailpipe.io/plugins/turbot/aws/sources/aws_s3_bucket#arguments): + +| Argument | Default | +|--------------|---------| +| file_layout | `AWSLogs/(%{DATA:org_id}/)?%{NUMBER:account_id}/SecurityHub/%{DATA:region_path}/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{DATA}.jsonl.gz` | \ No newline at end of file diff --git a/docs/tables/aws_securityhub_finding/queries.md b/docs/tables/aws_securityhub_finding/queries.md new file mode 100644 index 0000000..96d0b11 --- /dev/null +++ b/docs/tables/aws_securityhub_finding/queries.md @@ -0,0 +1,217 @@ +## Activity Examples + +### Daily Activity Trends + +Analyze the daily distribution of Security Hub findings to identify security patterns and potential security issues over time. + +```sql +select + strftime(tp_timestamp, '%Y-%m-%d') as finding_date, + count(*) as finding_count, + round(avg(severity.normalized), 2) as avg_severity +from + aws_securityhub_finding +group by + finding_date +order by + finding_date asc; +``` + +```yaml +folder: SecurityHub +``` + +### Recent Findings Analysis + +Analyze recent security findings with detailed resource and severity information. + +```sql +select + tp_timestamp, + title, + types, + severity, + resources, + tp_index as account_id, + region, + workflow_state, + remediation.recommendation.text as remediation_text +from + aws_securityhub_finding +where + tp_timestamp > current_date - interval '7 days' +order by + severity.normalized desc, + tp_timestamp desc; +``` + +```yaml +folder: SecurityHub +``` + +### Top 10 Finding Types + +Generate a ranked list of the most prevalent Security Hub finding types with severity information. + +```sql +select + types, + count(*) as finding_count, + round(avg(severity.normalized), 2) as avg_severity +from + aws_securityhub_finding +group by + types +order by + finding_count desc +limit 10; +``` + +```yaml +folder: SecurityHub +``` + + + +### Findings by Account and Region + +Analyze security findings across your AWS organization with detailed severity information. + +```sql +select + tp_index as account_id, + region, + count(*) as finding_count, + round(avg(severity.normalized), 2) as avg_severity, + sum(case when severity.normalized >= 90 then 1 else 0 end) as critical_severity_count, + sum(case when severity.normalized >= 70 and severity.normalized < 90 then 1 else 0 end) as high_severity_count, + sum(case when severity.normalized >= 40 and severity.normalized < 70 then 1 else 0 end) as medium_severity_count, + sum(case when severity.normalized >= 1 and severity.normalized < 40 then 1 else 0 end) as low_severity_count, + sum(case when severity.normalized = 0 then 1 else 0 end) as informational_severity_count +from + aws_securityhub_finding +group by + account_id, + region +order by + critical_severity_count desc; +``` + +```yaml +folder: SecurityHub +``` + +### Findings by Severity Level + +Categorize Security Hub findings into severity bands with detailed counts and percentages. + +```sql +select + case + when severity.normalized >= 90 then 'Critical (90-100)' + when severity.normalized >= 70 then 'High (70-89)' + when severity.normalized >= 40 then 'Medium (40-69)' + when severity.normalized >= 1 then 'Low (1-39)' + else 'Informational (0)' + end as severity_level, + count(*) as finding_count, + round(count(*) * 100.0 / sum(count(*)) over(), 2) as percentage +from + aws_securityhub_finding +group by + severity_level +order by + case severity_level + when 'Critical (90-100)' then 1 + when 'High (70-89)' then 2 + when 'Medium (40-69)' then 3 + when 'Low (1-39)' then 4 + else 5 + end; +``` + +```yaml +folder: SecurityHub +``` + +## Compliance Examples + +### Compliance Status Overview + +Monitor compliance status with detailed severity information. + +```sql +select + compliance.status, + compliance.security_control_id, + count(*) as finding_count, + round(avg(severity.normalized), 2) as avg_severity +from + aws_securityhub_finding +where + compliance is not null +group by + compliance.status, + compliance.security_control_id +order by + finding_count desc; +``` + +```yaml +folder: SecurityHub +``` + +## Detection Examples + +### Detect High Severity Findings with Remediation + + ```sql +select + tp_timestamp, + title, + types, + severity, + description, + tp_index as account_id, + region, + resources, + remediation.recommendation.text as remediation_text +from + aws_securityhub_finding +where + severity.normalized >= 70 +order by + severity.normalized desc, + tp_timestamp desc; +``` + +```yaml +folder: SecurityHub +``` + +### Lambda Function Security Issues + +Identify security issues in Lambda functions, focusing on public access. + +```sql +select + tp_timestamp, + title, + severity.normalized as severity, + json_extract(resources, '$[0].id') as function_arn, + json_extract(resources, '$[0].details.awslambdafunction.runtime') as runtime, + workflow_state +from + aws_securityhub_finding +where + json_extract(resources, '$[0].type') = '"AwsLambdaFunction"' + and title ilike '%public access%' + and severity.normalized >= 70 +order by + severity desc, + tp_timestamp desc; +``` + +```yaml +folder: SecurityHub +``` diff --git a/tables/securityhub_finding/securityhub_finding.go b/tables/securityhub_finding/securityhub_finding.go new file mode 100644 index 0000000..680435a --- /dev/null +++ b/tables/securityhub_finding/securityhub_finding.go @@ -0,0 +1,143 @@ +package securityhub_finding + +import ( + "time" + + "github.com/aws/aws-sdk-go-v2/service/securityhub/types" + "github.com/turbot/tailpipe-plugin-sdk/schema" +) + +type SecurityHubFinding struct { + schema.CommonFields + + // Top level fields + Version *string `json:"version,omitempty"` + ID *string `json:"id,omitempty"` + DetailType *string `json:"detail_type,omitempty"` + Source *string `json:"source,omitempty"` + Account *string `json:"account,omitempty"` + Time *time.Time `json:"time,omitempty"` + Region *string `json:"region,omitempty"` + + // Finding array schema + AwsAccountName *string `json:"aws_account_name" parquet:"name=aws_account_name"` + CompanyName *string `json:"company_name" parquet:"name=company_name"` + Compliance *types.Compliance `json:"compliance" parquet:"name=compliance"` + Confidence *int32 `json:"confidence" parquet:"name=confidence"` + CreatedAt *string `json:"createdAt" parquet:"name=created_at"` + Criticality *int32 `json:"criticality" parquet:"name=criticality"` + Description *string `json:"description" parquet:"name=description"` + FirstObservedAt *string `json:"first_observed_at" parquet:"name=first_observed_at"` + GeneratorId *string `json:"generatorId" parquet:"name=generator_id"` + GeneratorDetails *types.GeneratorDetails `json:"generator_details" parquet:"name=generator_details"` + FindingId *string `json:"findingId" parquet:"name=finding_id"` + FindingRegion *string `json:"findingRegion" parquet:"name=finding_region"` + LastObservedAt *string `json:"last_observed_at" parquet:"name=last_observed_at"` + Malware []types.Malware `json:"malware" parquet:"name=malware"` + Network *types.Network `json:"network" parquet:"name=network"` + NetworkPath []types.NetworkPathComponent `json:"network_path" parquet:"name=network_path"` + Note *types.Note `json:"note" parquet:"name=note"` + PatchSummary *types.PatchSummary `json:"patch_summary" parquet:"name=patch_summary"` + Process *types.ProcessDetails `json:"process" parquet:"name=process"` + ProcessedAt *string `json:"processed_at" parquet:"name=processed_at"` + ProductArn *string `json:"product_arn" parquet:"name=product_arn"` + ProductFields map[string]string `json:"product_fields" parquet:"name=product_fields"` + ProductName *string `json:"product_name" parquet:"name=product_name"` + RecordState types.RecordState `json:"record_state" parquet:"name=record_state"` + RelatedFindings []types.RelatedFinding `json:"related_findings" parquet:"name=related_findings"` + Remediation *types.Remediation `json:"remediation" parquet:"name=remediation"` + Resources []types.Resource `json:"resources" parquet:"name=resources"` + Action *types.Action `json:"action" parquet:"name=action"` + Sample *bool `json:"sample" parquet:"name=sample"` + SchemaVersion *string `json:"schema_version" parquet:"name=schema_version"` + Severity *types.Severity `json:"severity" parquet:"name=severity"` + SourceUrl *string `json:"source_url" parquet:"name=source_url"` + ThreatIntelIndicators []types.ThreatIntelIndicator `json:"threat_intel_indicators" parquet:"name=threat_intel_indicators"` + Threats []types.Threat `json:"threats" parquet:"name=threats"` + Title *string `json:"title" parquet:"name=title"` + Types []string `json:"types" parquet:"name=types"` + UpdatedAt *string `json:"updated_at" parquet:"name=updated_at"` + UserDefinedFields map[string]string `json:"user_defined_fields" parquet:"name=user_defined_fields"` + VerificationState types.VerificationState `json:"verification_state" parquet:"name=verification_state"` + Vulnerabilities []types.Vulnerability `json:"vulnerabilities" parquet:"name=vulnerabilities"` + Workflow *types.Workflow `json:"workflow" parquet:"name=workflow"` + WorkflowState types.WorkflowState `json:"workflow_state" parquet:"name=workflow_state"` +} + +// DetailFindingsData maps the `detail` field containing findings +// The following struct will be used for only parse the log lines +type DetailFindingsData struct { + Version *string `json:"version,omitempty"` + ID *string `json:"id,omitempty"` + DetailType *string `json:"detail_type,omitempty"` + Source *string `json:"source,omitempty"` + Account *string `json:"account,omitempty"` + Time *time.Time `json:"time,omitempty"` + Region *string `json:"region,omitempty"` + Detail struct { + Findings []types.AwsSecurityFinding `json:"findings" parquet:"name=findings, type=JSON"` + } `json:"detail" parquet:"name=detail, type=JSON"` +} + +func (c *SecurityHubFinding) GetColumnDescriptions() map[string]string { + return map[string]string{ + // Top level fields + "version": "The version of the event format.", + "id": "The unique identifier for the event.", + "detail_type": "The type of the event detail.", + "source": "The service or system that generated the event.", + "account": "The AWS account ID where the finding was generated.", + "time": "The timestamp when the event was generated.", + "region": "The AWS region where the finding was generated.", + + // Finding fields + "aws_account_name": "The name of the AWS account from which a finding was generated.", + "company_name": "The name of the company for the product that generated the finding. Security Hub populates this attribute automatically for each finding.", + "compliance": "Contains security standard-related finding details for findings generated from compliance checks against specific rules in supported security standards.", + "confidence": "The likelihood that a finding accurately identifies the behavior or issue that it was intended to identify. Scored on a 0-100 basis.", + "created_at": "The timestamp when the security findings provider created the potential security issue that a finding captured.", + "criticality": "The level of importance assigned to the resources associated with the finding. A score of 0 means no criticality, and 100 is reserved for the most critical resources.", + "description": "A detailed description of the security finding.", + "first_observed_at": "The timestamp when the security findings provider first observed the potential security issue that a finding captured.", + "generator_id": "The identifier for the solution-specific component (a discrete unit of logic) that generated a finding.", + "generator_details": "Metadata for the Amazon CodeGuru detector associated with a finding, particularly for Lambda function-related findings.", + "finding_id": "The security findings provider-specific identifier for a finding.", + "finding_region": "The AWS region from which the finding was generated.", + "last_observed_at": "The timestamp when the security findings provider most recently observed the potential security issue that a finding captured.", + "malware": "A list of malware related to a finding.", + "network": "The details of network-related information about a finding.", + "network_path": "Information about a network path that is relevant to a finding, with each entry representing a component of that path.", + "note": "A user-defined note added to a finding.", + "patch_summary": "An overview of the patch compliance status for an instance against a selected compliance standard.", + "process": "The details of process-related information about a finding.", + "processed_at": "The timestamp when Security Hub received a finding and began to process it.", + "product_arn": "The ARN generated by Security Hub that uniquely identifies a product that generates findings.", + "product_fields": "Additional solution-specific details that aren't part of the defined AwsSecurityFinding format. Can contain up to 50 key-value pairs.", + "product_name": "The name of the product that generated the finding. Security Hub populates this attribute automatically for each finding.", + "record_state": "The record state of a finding.", + "related_findings": "A list of related findings.", + "remediation": "A data type that describes the remediation options for a finding.", + "resources": "A set of resource data types that describe the resources that the finding refers to.", + "action": "Details about an action that affects or that was taken on a resource.", + "sample": "Indicates whether the finding is a sample finding.", + "schema_version": "The schema version that a finding is formatted for.", + "severity": "The severity level of the finding.", + "source_url": "A URL that links to a page about the current finding in the security findings provider's solution.", + "threat_intel_indicators": "Threat intelligence details related to a finding.", + "threats": "Details about the threat detected in a security finding and the file paths that were affected by the threat.", + "title": "A short human-readable title for the finding.", + "types": "One or more finding types in the format of namespace/category/classifier that classify a finding.", + "updated_at": "The timestamp when the security findings provider last updated the finding record.", + "user_defined_fields": "A list of name/value string pairs associated with the finding. These are custom, user-defined fields added to a finding.", + "verification_state": "Indicates the veracity of a finding.", + "vulnerabilities": "A list of vulnerabilities associated with the findings.", + "workflow": "Information about the status of the investigation into a finding.", + "workflow_state": "The workflow state of a finding.", + + // Tailpipe-specific metadata fields + "tp_akas": "The list of AWS ARNs associated with the finding.", + "tp_index": "The AWS account ID where the finding was generated.", + "tp_timestamp": "The timestamp when the finding was generated.", + "tp_date": "The date when the finding was generated, truncated to day.", + } +} diff --git a/tables/securityhub_finding/securityhub_finding_extractor.go b/tables/securityhub_finding/securityhub_finding_extractor.go new file mode 100644 index 0000000..a13de40 --- /dev/null +++ b/tables/securityhub_finding/securityhub_finding_extractor.go @@ -0,0 +1,239 @@ +package securityhub_finding + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + + "github.com/turbot/tailpipe-plugin-sdk/artifact_source" +) + +// SecurityHubFindingExtractor is an extractor that receives JSON serialised SecurityHub findings +// and extracts SecurityHubFinding records from them +type SecurityHubFindingExtractor struct { +} + +// NewSecurityHubFindingExtractor creates a new SecurityHubFindingExtractor +func NewSecurityHubFindingExtractor() artifact_source.Extractor { + return &SecurityHubFindingExtractor{} +} + +func (c *SecurityHubFindingExtractor) Identifier() string { + return "securityhub_finding_extractor" +} + +// Extract unmarshalls the artifact data as SecurityHub findings and returns the SecurityHubFinding records +func (c *SecurityHubFindingExtractor) Extract(_ context.Context, a any) ([]any, error) { + // the expected input type is a JSON byte[] deserializable to DetailFindingsData + var jsonBytes []byte + + switch v := a.(type) { + case []byte: + jsonBytes = v + case string: + jsonBytes = []byte(v) + default: + return nil, fmt.Errorf("expected []byte or string, got %T", a) + } + + // First, we need to remap certain JSON fields due to naming conventions + // DetailFindingsData expects "detail-type" to be mapped to "detail_type" + var rawEvent map[string]json.RawMessage + if err := json.Unmarshal(jsonBytes, &rawEvent); err != nil { + return nil, fmt.Errorf("error decoding json: %w", err) + } + + // Handle kebab-case to snake_case for detail-type + if detailType, ok := rawEvent["detail-type"]; ok { + rawEvent["detail_type"] = detailType + delete(rawEvent, "detail-type") + } + + // Re-encode the modified JSON + modifiedJSON, err := json.Marshal(rawEvent) + if err != nil { + return nil, fmt.Errorf("error re-encoding json: %w", err) + } + + // decode json into DetailFindingsData + var event DetailFindingsData + err = json.Unmarshal(modifiedJSON, &event) + if err != nil { + slog.Debug("Error decoding SecurityHub finding", "error", err, "sample_start", string(jsonBytes[:min(len(jsonBytes), 500)])) + return nil, fmt.Errorf("error decoding json: %w", err) + } + + slog.Debug("SecurityHubFindingExtractor", "record count", len(event.Detail.Findings)) + + findings := toMapSecurityHubFinding(event) + var res = make([]any, len(findings)) + for i, record := range findings { + res[i] = &record + } + return res, nil +} + +func toMapSecurityHubFinding(event DetailFindingsData) []SecurityHubFinding { + var findings []SecurityHubFinding + + for _, finding := range event.Detail.Findings { + f := SecurityHubFinding{} + + // Event metadata + f.Version = event.Version + f.ID = event.ID + f.DetailType = event.DetailType + f.Source = event.Source + f.Account = event.Account + f.Time = event.Time + f.Region = event.Region + + // Finding details from AWS security finding + if finding.AwsAccountName != nil { + f.AwsAccountName = finding.AwsAccountName + } + if finding.CompanyName != nil { + f.CompanyName = finding.CompanyName + } + if finding.Compliance != nil { + f.Compliance = finding.Compliance + } + if finding.Confidence != nil { + f.Confidence = finding.Confidence + } + if finding.CreatedAt != nil { + createdAtStr := *finding.CreatedAt + f.CreatedAt = &createdAtStr + } + if finding.Criticality != nil { + f.Criticality = finding.Criticality + } + if finding.Description != nil { + f.Description = finding.Description + } + if finding.FirstObservedAt != nil { + f.FirstObservedAt = finding.FirstObservedAt + } + if finding.GeneratorId != nil { + f.GeneratorId = finding.GeneratorId + } + if finding.GeneratorDetails != nil { + f.GeneratorDetails = finding.GeneratorDetails + } + if finding.Id != nil { + f.FindingId = finding.Id + } + if finding.Region != nil { + f.FindingRegion = finding.Region + } + if finding.LastObservedAt != nil { + f.LastObservedAt = finding.LastObservedAt + } + if finding.Malware != nil { + f.Malware = finding.Malware + } + if finding.Network != nil { + f.Network = finding.Network + } + if finding.NetworkPath != nil { + f.NetworkPath = finding.NetworkPath + } + if finding.Note != nil { + f.Note = finding.Note + } + if finding.PatchSummary != nil { + f.PatchSummary = finding.PatchSummary + } + if finding.Process != nil { + f.Process = finding.Process + } + if finding.ProcessedAt != nil { + f.ProcessedAt = finding.ProcessedAt + } + if finding.ProductArn != nil { + f.ProductArn = finding.ProductArn + } + if finding.ProductName != nil { + f.ProductName = finding.ProductName + } + if finding.RecordState != "" { + f.RecordState = finding.RecordState + } + if finding.RelatedFindings != nil { + f.RelatedFindings = finding.RelatedFindings + } + if finding.Action != nil { + f.Action = finding.Action + } + if finding.Sample != nil { + f.Sample = finding.Sample + } + if finding.SchemaVersion != nil { + f.SchemaVersion = finding.SchemaVersion + } + if finding.Severity != nil { + f.Severity = finding.Severity + } + if finding.SourceUrl != nil { + f.SourceUrl = finding.SourceUrl + } + if finding.ThreatIntelIndicators != nil { + f.ThreatIntelIndicators = finding.ThreatIntelIndicators + } + if finding.Threats != nil { + f.Threats = finding.Threats + } + if finding.Title != nil { + f.Title = finding.Title + } + if finding.Types != nil { + f.Types = finding.Types + } + if finding.UpdatedAt != nil { + f.UpdatedAt = finding.UpdatedAt + } + if finding.UserDefinedFields != nil { + userDefinedFields := make(map[string]string) + for k, v := range finding.UserDefinedFields { + userDefinedFields[k] = v + } + f.UserDefinedFields = userDefinedFields + } + if finding.VerificationState != "" { + f.VerificationState = finding.VerificationState + } + if finding.Vulnerabilities != nil { + f.Vulnerabilities = finding.Vulnerabilities + } + if finding.Workflow != nil { + f.Workflow = finding.Workflow + } + if finding.WorkflowState != "" { + f.WorkflowState = finding.WorkflowState + } + + // Map ProductFields + if finding.ProductFields != nil { + productFields := make(map[string]string) + for k, v := range finding.ProductFields { + productFields[k] = v + } + f.ProductFields = productFields + } + + // Map Resources + if len(finding.Resources) > 0 { + f.Resources = finding.Resources + } + + // Map Remediation + if finding.Remediation != nil && finding.Remediation.Recommendation != nil { + f.Remediation = finding.Remediation + } + + findings = append(findings, f) + } + + return findings +} diff --git a/tables/securityhub_finding/securityhub_finding_mapper.go b/tables/securityhub_finding/securityhub_finding_mapper.go new file mode 100644 index 0000000..4b761cb --- /dev/null +++ b/tables/securityhub_finding/securityhub_finding_mapper.go @@ -0,0 +1,42 @@ +package securityhub_finding + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/turbot/tailpipe-plugin-sdk/mappers" +) + +type SecurityHubFindingMapper struct { +} + +func (m *SecurityHubFindingMapper) Identifier() string { + return "security_hub_finding_mapper" +} + +func (m *SecurityHubFindingMapper) Map(_ context.Context, a any, _ ...mappers.MapOption[*SecurityHubFinding]) (*SecurityHubFinding, error) { + var b SecurityHubFinding + + switch data := a.(type) { + case []byte: + if err := json.Unmarshal(data, &b); err != nil { + return nil, fmt.Errorf("error unmarshalling row data: %w", err) + } + case string: + if err := json.Unmarshal([]byte(data), &b); err != nil { + return nil, fmt.Errorf("error unmarshalling row data: %w", err) + } + case SecurityHubFinding: + b = data + return &b, nil + case *SecurityHubFinding: + b = *data + return &b, nil + default: + return nil, fmt.Errorf("expected byte[], string or SecurityHubFinding, got %T", a) + } + + return &b, nil + +} \ No newline at end of file diff --git a/tables/securityhub_finding/securityhub_finding_table.go b/tables/securityhub_finding/securityhub_finding_table.go new file mode 100644 index 0000000..478ee4a --- /dev/null +++ b/tables/securityhub_finding/securityhub_finding_table.go @@ -0,0 +1,75 @@ +package securityhub_finding + +import ( + "time" + + "github.com/rs/xid" + "github.com/turbot/pipe-fittings/v2/utils" + "github.com/turbot/tailpipe-plugin-aws/sources/s3_bucket" + "github.com/turbot/tailpipe-plugin-aws/tables" + "github.com/turbot/tailpipe-plugin-sdk/artifact_source" + "github.com/turbot/tailpipe-plugin-sdk/artifact_source_config" + "github.com/turbot/tailpipe-plugin-sdk/constants" + "github.com/turbot/tailpipe-plugin-sdk/row_source" + "github.com/turbot/tailpipe-plugin-sdk/schema" + "github.com/turbot/tailpipe-plugin-sdk/table" +) + +const SecurityHubFindingTableIdentifier = "aws_securityhub_finding" + +type SecurityHubFindingTable struct{} + +func (c *SecurityHubFindingTable) Identifier() string { + return SecurityHubFindingTableIdentifier +} + +func (c *SecurityHubFindingTable) GetSourceMetadata() ([]*table.SourceMetadata[*SecurityHubFinding], error) { + defaultS3ArtifactConfig := &artifact_source_config.ArtifactSourceConfigImpl{ + FileLayout: utils.ToStringPointer("AWSLogs/(%{DATA:org_id}/)?%{NUMBER:account_id}/SecurityHub/%{DATA:region}/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{DATA}.json.gz"), + } + + return []*table.SourceMetadata[*SecurityHubFinding]{ + { + // S3 artifact source + SourceName: s3_bucket.AwsS3BucketSourceIdentifier, + Mapper: &SecurityHubFindingMapper{}, + Options: []row_source.RowSourceOption{ + artifact_source.WithDefaultArtifactSourceConfig(defaultS3ArtifactConfig), + artifact_source.WithArtifactExtractor(NewSecurityHubFindingExtractor()), + }, + }, + { + SourceName: constants.ArtifactSourceIdentifier, + Mapper: &SecurityHubFindingMapper{}, + Options: []row_source.RowSourceOption{ + artifact_source.WithArtifactExtractor(NewSecurityHubFindingExtractor()), + }, + }, + }, nil +} + +func (c *SecurityHubFindingTable) EnrichRow(row *SecurityHubFinding, sourceEnrichmentFields schema.SourceEnrichment) (*SecurityHubFinding, error) { + row.CommonFields = sourceEnrichmentFields.CommonFields + + row.TpID = xid.New().String() + row.TpIngestTimestamp = time.Now() + + if row.ProductArn != nil { + akas := tables.AwsAkasFromArn(*row.ProductArn) + row.TpAkas = append(row.TpAkas, akas...) + } + + if row.Time != nil { + row.TpTimestamp = *row.Time + row.TpDate = row.Time.Truncate(24 * time.Hour) + } + if row.Account != nil { + row.TpIndex = *row.Account + } + + return row, nil +} + +func (c *SecurityHubFindingTable) GetDescription() string { + return "AWS Security Hub findings provide detailed information about potential security issues and compliance violations detected across your AWS accounts and resources. This table captures comprehensive security findings from various AWS security services and partner integrations, including details about the affected resources, severity levels, compliance status, and recommended remediation steps." +}