turbot · ParthaI · Aug 12, 2025 · Aug 22, 2025 · Aug 22, 2025 · Aug 22, 2025
diff --git a/config/gcp_connection.go b/config/gcp_connection.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"time"
 
 	"github.com/mitchellh/go-homedir"
 	"golang.org/x/oauth2"
@@ -19,9 +20,28 @@ type GcpConnection struct {
 	QuotaProject              *string `json:"quota_project" hcl:"quota_project"`
 	ImpersonateAccessToken    *string `json:"impersonate_access_token" hcl:"impersonate_access_token"`
 	ImpersonateServiceAccount *string `json:"impersonate_service_account" hcl:"impersonate_service_account"`
+
+	// Retry configuration for GAPIC clients
+	// These parameters control the exponential backoff retry mechanism
+	MinRetryDelay     *int     `hcl:"min_retry_delay,optional" json:"min_retry_delay"`       // Initial retry delay in milliseconds (default: 500ms)
+	MaxRetryDelay     *int     `hcl:"max_retry_delay,optional" json:"max_retry_delay"`       // Maximum retry delay in milliseconds (default: 60000ms)
+	BackoffMultiplier *float64 `hcl:"backoff_multiplier,optional" json:"backoff_multiplier"` // Exponential growth multiplier (default: 1.30)
 }
 
 func (c *GcpConnection) Validate() error {
+	// Validate retry configuration
+	if c.MinRetryDelay != nil && *c.MinRetryDelay < 1 {
+		return fmt.Errorf("min_retry_delay must be greater than or equal to 1, got %d", *c.MinRetryDelay)
+	}
+
+	if c.MaxRetryDelay != nil && *c.MaxRetryDelay < 1 {
+		return fmt.Errorf("max_retry_delay must be greater than or equal to 1, got %d", *c.MaxRetryDelay)
+	}
+
+	if c.BackoffMultiplier != nil && *c.BackoffMultiplier <= 0 {
+		return fmt.Errorf("backoff_multiplier must be greater than 0, got %f", *c.BackoffMultiplier)
+	}
+
 	return nil
 }
 
@@ -95,6 +115,28 @@ func (c *GcpConnection) GetClientOptions(ctx context.Context) ([]option.ClientOp
 	return opts, nil
 }
 
+// GetRetryConfig returns the retry configuration for GAPIC clients
+func (c *GcpConnection) GetRetryConfig() (minDelay, maxDelay time.Duration, multiplier float64) {
+	// Default values as per GCP SDK
+	// https://github.com/googleapis/google-cloud-go/blob/logging/v1.13.0/logging/apiv2/logging_client.go#L105-L117
+	minDelay = 500 * time.Millisecond
+	maxDelay = 60000 * time.Millisecond
+	multiplier = 1.30
+
+	// Check connection config
+	if c.MinRetryDelay != nil {
+		minDelay = time.Duration(*c.MinRetryDelay) * time.Millisecond
+	}
+	if c.MaxRetryDelay != nil {
+		maxDelay = time.Duration(*c.MaxRetryDelay) * time.Millisecond
+	}
+	if c.BackoffMultiplier != nil {
+		multiplier = *c.BackoffMultiplier
+	}
+
+	return minDelay, maxDelay, multiplier
+}
+
 // TODO: #graza #refactor Determine where this actually belongs, maybe a useful util func? https://github.com/turbot/tailpipe-plugin-gcp/issues/17
 func (c *GcpConnection) pathOrContents(in string) (string, error) {
 	if len(in) == 0 {

diff --git a/docs/index.md b/docs/index.md
@@ -117,13 +117,16 @@ To get started, choose a mod from the [Powerpipe Hub](https://hub.powerpipe.io/?
 
 ### Arguments
 
-| Name                          | Type   | Required | Description                                                                                          |
-|-------------------------------|--------|----------|------------------------------------------------------------------------------------------------------|
-| `credentials`                 | String | No       | Path to the JSON credentials file or the contents of a service account key file in JSON format.     |
-| `impersonate_access_token`    | String | No       | An OAuth 2.0 access token used to impersonate a service account.                                    |
-| `impersonate_service_account` | String | No       | The email of the service account to impersonate for authentication.                                |
-| `project`                     | String | No       | The project ID to connect to.                                                                      |
-| `quota_project`               | String | No       | The project ID to use for quota usage and billing purposes.                                        |
+| Name                          | Type   | Required | Description                                                                                     |
+| ----------------------------- | ------ | -------- | ----------------------------------------------------------------------------------------------- |
+| `credentials`                 | String | No       | Path to the JSON credentials file or the contents of a service account key file in JSON format. |
+| `impersonate_access_token`    | String | No       | An OAuth 2.0 access token used to impersonate a service account.                                |
+| `impersonate_service_account` | String | No       | The email of the service account to impersonate for authentication.                             |
+| `project`                     | String | No       | The project ID to connect to.                                                                   |
+| `quota_project`               | String | No       | The project ID to use for quota usage and billing purposes.                                     |
+| `min_retry_delay`             | Int    | No       | Initial retry delay in milliseconds for API rate limiting (default: 500ms).                     |
+| `max_retry_delay`             | Int    | No       | Maximum retry delay in milliseconds for API rate limiting (default: 60000ms).                   |
+| `backoff_multiplier`          | Float  | No       | Exponential growth multiplier for retry delays (default: 1.30).                                 |
 
 ### Application Default Credentials
 

diff --git a/docs/sources/gcp_audit_log_api.md b/docs/sources/gcp_audit_log_api.md
diff --git a/docs/sources/gcp_logging_log_entry.md b/docs/sources/gcp_logging_log_entry.md
@@ -0,0 +1,77 @@
+---
+title: "Source: gcp_logging_log_entry - Collect logs from GCP logging API"
+description: "Allows users to collect logs from Google Cloud Platform (GCP) logging API."
+---
+
+# Source: gcp_logging_log_entry - Obtain logs from GCP logging API
+
+The Google Cloud Platform (GCP) logging API provides access to various types of logs for GCP services. It allows you to view and manage logs for your GCP projects, including logs for administrative actions, data access, system events, request logs, and more.
+
+Using this source, you can collect, filter, and analyze logs retrieved from the GCP logging API, enabling system monitoring, security investigations, and compliance reporting.
+
+## Automatic Table Detection
+
+The source automatically detects which table is being used based on the partition configuration. When you run a command like:
+
+```bash
+tpc gcp_audit_log.my_logs --from T-30d
+```
+
+The source will:
+
+1. Automatically detect the table name (`gcp_audit_log`) from the partition configuration
+2. Automatically use the appropriate log types for that table
+3. For `gcp_audit_log` table: uses `activity`, `data_access`, `system_event`, and `policy` logs
+
+This means you don't need to specify log types manually - the source will automatically filter logs based on the table you're collecting for.
+
+## Example Configurations
+
+### Collect all types of logs
+
+Collect all types of logs for a project.
+
+```hcl
+connection "gcp" "my_project" {
+  project = "my-gcp-project"
+}
+
+partition "gcp_audit_log" "my_logs" {
+  source "gcp_logging_log_entry" {
+    connection = connection.gcp.my_project
+  }
+}
+```
+
+### Collect specific types of logs
+
+Collect admin activity and data access logs for a project.
+
+```hcl
+partition "gcp_audit_log" "my_logs_admin_data_access" {
+  source "gcp_logging_log_entry" {
+    connection = connection.gcp.my_project
+    log_types = ["activity", "data_access"]
+  }
+}
+```
+
+### Collect specific log type
+
+Collect only activity logs.
+
+```hcl
+partition "gcp_audit_log" "my_activity_logs" {
+  source "gcp_logging_log_entry" {
+    connection = connection.gcp.my_project
+    log_types = ["activity"]
+  }
+}
+```
+
+## Arguments
+
+| Argument   | Type             | Required | Default                  | Description                                                                                                                                                       |
+| ---------- | ---------------- | -------- | ------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| connection | `connection.gcp` | No       | `connection.gcp.default` | The [GCP connection](https://hub.tailpipe.io/plugins/turbot/gcp#connection-credentials) to use to connect to the GCP account.                                     |
+| log_types  | List(String)     | No       | []                       | A list of log types to retrieve. If no types are specified, all log types for the table are retrieved. Valid values: activity, data_access, system_event, policy. |
diff --git a/docs/tables/gcp_audit_log/index.md b/docs/tables/gcp_audit_log/index.md
@@ -154,29 +154,29 @@ partition "gcp_audit_log" "my_logs_prefix" {
 }
 ```
 
-### Collect logs from audit logs API
+### Collect logs from logging API
 
-Collect audit logs stored in a Storage bucket that use the [default log file name format](https://hub.tailpipe.io/plugins/turbot/gcp/tables/gcp_audit_log#gcp_storage_bucket).
+Collect audit logs using the GCP logging API.
 
 ```hcl
 connection "gcp" "my_project" {
   project = "my-gcp-project"
 }
 
 partition "gcp_audit_log" "my_logs" {
-  source "gcp_audit_log_api" {
+  source "logging_log_entry" {
     connection = connection.gcp.my_project
   }
 }
 ```
 
-### Collect specific types of audit logs from audit logs API
+### Collect specific types of audit logs from logging API
 
 Collect admin activity and data access audit logs for a project.
 
 ```hcl
 partition "gcp_audit_log" "my_logs_admin_data_access" {
-  source "gcp_audit_log_api" {
+  source "logging_log_entry" {
     connection = connection.gcp.my_project
     log_types = ["activity", "data_access"]
   }
@@ -205,6 +205,6 @@ partition "gcp_audit_log" "my_logs_severity" {
 
 This table sets the following defaults for the [gcp_storage_bucket](https://hub.tailpipe.io/plugins/turbot/gcp/sources/gcp_storage_bucket#arguments):
 
-| Argument      | Default |
-|--------------|---------|
-| file_layout   | `cloudaudit.googleapis.com/%{DATA:type}/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{HOUR:hour}:%{MINUTE:minute}:%{SECOND:second}_%{DATA:end_time}_%{DATA:suffix}.json` |
+| Argument    | Default                                                                                                                                                                     |
+| ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| file_layout | `cloudaudit.googleapis.com/%{DATA:type}/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{HOUR:hour}:%{MINUTE:minute}:%{SECOND:second}_%{DATA:end_time}_%{DATA:suffix}.json` |
diff --git a/gcp/plugin.go b/gcp/plugin.go
@@ -3,7 +3,7 @@ package gcp
 import (
 	"github.com/turbot/go-kit/helpers"
 	"github.com/turbot/tailpipe-plugin-gcp/config"
-	"github.com/turbot/tailpipe-plugin-gcp/sources/audit_log_api"
+	logging_log_entry "github.com/turbot/tailpipe-plugin-gcp/sources/logging_log_entry"
 	"github.com/turbot/tailpipe-plugin-gcp/sources/storage_bucket"
 	"github.com/turbot/tailpipe-plugin-gcp/tables/audit_log"
 	"github.com/turbot/tailpipe-plugin-sdk/plugin"
@@ -22,7 +22,7 @@ func init() {
 	table.RegisterTable[*audit_log.AuditLog, *audit_log.AuditLogTable]()
 
 	// register sources
-	row_source.RegisterRowSource[*audit_log_api.AuditLogAPISource]()
+	row_source.RegisterRowSource[*logging_log_entry.LoggingLogEntrySource]()
 	row_source.RegisterRowSource[*storage_bucket.GcpStorageBucketSource]()
 }
 

diff --git a/go.mod b/go.mod
@@ -8,6 +8,7 @@ require (
 	cloud.google.com/go/logging v1.13.0
 	cloud.google.com/go/storage v1.54.0
 	github.com/elastic/go-grok v0.3.1
+	github.com/googleapis/gax-go/v2 v2.14.1
 	github.com/hashicorp/hcl/v2 v2.20.1
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/rs/xid v1.6.0
@@ -17,6 +18,7 @@ require (
 	golang.org/x/oauth2 v0.30.0
 	google.golang.org/api v0.232.0
 	google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb
+	google.golang.org/grpc v1.72.0
 	google.golang.org/protobuf v1.36.6
 )
 
@@ -98,7 +100,6 @@ require (
 	github.com/google/s2a-go v0.1.9 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
-	github.com/googleapis/gax-go/v2 v2.14.1 // indirect
 	github.com/gopherjs/gopherjs v1.17.2 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
@@ -210,7 +211,6 @@ require (
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20250505200425-f936aa4a68b2 // indirect
-	google.golang.org/grpc v1.72.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	oras.land/oras-go/v2 v2.5.0 // indirect