Merge branch 'master' into use-workers-analyzer

.github/workflows/go-ci.yml

@@ -51,12 +51,6 @@ jobs:
         uses: actions/checkout@v3
         with:
           persist-credentials: false
-      - uses: dorny/paths-filter@v2
-        id: changes
-        with:
-          filters: |
-            src:
-              - 'pkg/terraformer/**'
       - name: Get cache paths
         id: go-cache-paths
         shell: bash

.grype.yaml

@@ -3,8 +3,6 @@ ignore:
       name: anchore/scan-action
   - package:
       location: "/root/.terraform.d/plugins/linux_amd64/terraform-provider-**"
-  - package:
-      location: "/usr/bin/terraformer"
   - package:
       location: "/usr/bin/terraform"
   - package:

Dockerfile

@@ -43,24 +43,6 @@ RUN apk update --no-cache \
     gcompat~=1.1.0 \
     git~=2.43
 
-# Install Terraform and Terraform plugins
-RUN wget https://releases.hashicorp.com/terraform/1.5.6/terraform_1.5.6_linux_amd64.zip \
-    && unzip terraform_1.5.6_linux_amd64.zip && rm terraform_1.5.6_linux_amd64.zip \
-    && mv terraform /usr/bin/terraform \
-    && wget https://releases.hashicorp.com/terraform-provider-azurerm/3.71.0/terraform-provider-azurerm_3.71.0_linux_amd64.zip \
-    && wget https://releases.hashicorp.com/terraform-provider-aws/3.72.0/terraform-provider-aws_3.72.0_linux_amd64.zip \
-    && wget https://releases.hashicorp.com/terraform-provider-google/4.32.0/terraform-provider-google_4.32.0_linux_amd64.zip \
-    && unzip terraform-provider-azurerm_3.71.0_linux_amd64.zip && rm terraform-provider-azurerm_3.71.0_linux_amd64.zip\
-    && unzip terraform-provider-google_4.32.0_linux_amd64.zip && rm terraform-provider-google_4.32.0_linux_amd64.zip \
-    && unzip terraform-provider-aws_3.72.0_linux_amd64.zip && rm terraform-provider-aws_3.72.0_linux_amd64.zip \
-    && mkdir ~/.terraform.d && mkdir ~/.terraform.d/plugins && mkdir ~/.terraform.d/plugins/linux_amd64 && mv terraform-provider-aws_v3.72.0_x5 terraform-provider-google_v4.32.0_x5 terraform-provider-azurerm_v3.71.0_x5 ~/.terraform.d/plugins/linux_amd64
-
-# Install Terraformer
-RUN wget https://github.com/GoogleCloudPlatform/terraformer/releases/download/0.8.24/terraformer-all-linux-amd64 \
-    && chmod +x terraformer-all-linux-amd64 \
-    && mv terraformer-all-linux-amd64 /usr/bin/terraformer
-
-
 # Copy built binary to the runtime container
 # Vulnerability fixed in latest version of KICS remove when gh actions version is updated
 # kics-scan ignore-line

docker/Dockerfile.apispec

@@ -33,20 +33,6 @@ FROM alpine:3.15.4
 
 RUN addgroup -S Checkmarx && adduser -S Checkmarx -G Checkmarx
 
-# Install Terraform and Terraform plugins
-RUN wget https://releases.hashicorp.com/terraform/1.1.3/terraform_1.1.3_linux_amd64.zip
-RUN unzip terraform_1.1.3_linux_amd64.zip && rm terraform_1.1.3_linux_amd64.zip
-RUN mv terraform /usr/bin/terraform
-
-RUN wget https://releases.hashicorp.com/terraform-provider-azurerm/2.95.0/terraform-provider-azurerm_2.95.0_linux_amd64.zip
-RUN wget https://releases.hashicorp.com/terraform-provider-aws/3.72.0/terraform-provider-aws_3.72.0_linux_amd64.zip
-RUN wget https://releases.hashicorp.com/terraform-provider-google/4.10.0/terraform-provider-google_4.10.0_linux_amd64.zip
-RUN unzip terraform-provider-azurerm_2.95.0_linux_amd64.zip && rm terraform-provider-azurerm_2.95.0_linux_amd64.zip
-RUN unzip terraform-provider-google_4.10.0_linux_amd64.zip && rm terraform-provider-google_4.10.0_linux_amd64.zip
-RUN unzip terraform-provider-aws_3.72.0_linux_amd64.zip && rm terraform-provider-aws_3.72.0_linux_amd64.zip
-RUN mkdir ~/.terraform.d && mkdir ~/.terraform.d/plugins && mkdir ~/.terraform.d/plugins/linux_amd64 && mv terraform-provider-aws_v3.72.0_x5 terraform-provider-google_v4.10.0_x5 terraform-provider-azurerm_v2.95.0_x5 ~/.terraform.d/plugins/linux_amd64
-
-
 # Install Git
 RUN apk add --no-cache \
   git=2.36.3-r0

docker/Dockerfile.apispec.debian

@@ -51,20 +51,6 @@ RUN apt-get update -yq \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
 
-# Install Terraform and Terraform plugins
-RUN wget https://releases.hashicorp.com/terraform/1.1.3/terraform_1.1.3_linux_amd64.zip
-RUN unzip terraform_1.1.3_linux_amd64.zip && rm terraform_1.1.3_linux_amd64.zip
-RUN mv terraform /usr/bin/terraform
-
-RUN wget https://releases.hashicorp.com/terraform-provider-azurerm/2.95.0/terraform-provider-azurerm_2.95.0_linux_amd64.zip
-RUN wget https://releases.hashicorp.com/terraform-provider-aws/3.72.0/terraform-provider-aws_3.72.0_linux_amd64.zip
-RUN wget https://releases.hashicorp.com/terraform-provider-google/4.10.0/terraform-provider-google_4.10.0_linux_amd64.zip
-RUN unzip terraform-provider-azurerm_2.95.0_linux_amd64.zip && rm terraform-provider-azurerm_2.95.0_linux_amd64.zip
-RUN unzip terraform-provider-google_4.10.0_linux_amd64.zip && rm terraform-provider-google_4.10.0_linux_amd64.zip
-RUN unzip terraform-provider-aws_3.72.0_linux_amd64.zip && rm terraform-provider-aws_3.72.0_linux_amd64.zip
-RUN mkdir ~/.terraform.d && mkdir ~/.terraform.d/plugins && mkdir ~/.terraform.d/plugins/linux_amd64 && mv terraform-provider-aws_v3.72.0_x5 terraform-provider-google_v4.10.0_x5 terraform-provider-azurerm_v2.95.0_x5 ~/.terraform.d/plugins/linux_amd64
-
-
 COPY --from=build_env /app/bin/kics /app/bin/kics
 COPY --from=build_env /app/assets/libraries/common /app/bin/assets/libraries/common
 COPY --from=build_env /app/assets/libraries/openapi /app/bin/assets/libraries/openapi

docker/Dockerfile.debian

@@ -52,20 +52,6 @@ RUN apt-get update -yq \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
 
-# Install Terraform and Terraform plugins
-RUN wget https://releases.hashicorp.com/terraform/1.1.3/terraform_1.1.3_linux_amd64.zip
-RUN unzip terraform_1.1.3_linux_amd64.zip && rm terraform_1.1.3_linux_amd64.zip
-RUN mv terraform /usr/bin/terraform
-
-RUN wget https://releases.hashicorp.com/terraform-provider-azurerm/2.95.0/terraform-provider-azurerm_2.95.0_linux_amd64.zip
-RUN wget https://releases.hashicorp.com/terraform-provider-aws/3.72.0/terraform-provider-aws_3.72.0_linux_amd64.zip
-RUN wget https://releases.hashicorp.com/terraform-provider-google/4.10.0/terraform-provider-google_4.10.0_linux_amd64.zip
-RUN unzip terraform-provider-azurerm_2.95.0_linux_amd64.zip && rm terraform-provider-azurerm_2.95.0_linux_amd64.zip
-RUN unzip terraform-provider-google_4.10.0_linux_amd64.zip && rm terraform-provider-google_4.10.0_linux_amd64.zip
-RUN unzip terraform-provider-aws_3.72.0_linux_amd64.zip && rm terraform-provider-aws_3.72.0_linux_amd64.zip
-RUN mkdir ~/.terraform.d && mkdir ~/.terraform.d/plugins && mkdir ~/.terraform.d/plugins/linux_amd64 && mv terraform-provider-aws_v3.72.0_x5 terraform-provider-google_v4.10.0_x5 terraform-provider-azurerm_v2.95.0_x5 ~/.terraform.d/plugins/linux_amd64
-
-
 COPY --from=build_env /app/bin/kics /app/bin/kics
 COPY --from=build_env /app/assets/queries /app/bin/assets/queries
 COPY --from=build_env /app/assets/cwe_csv /app/bin/assets/cwe_csv

docker/Dockerfile.ubi8

@@ -71,23 +71,6 @@ COPY LICENSE /licenses/LICENSE
 
 WORKDIR /app/bin
 
-USER root
-
-# Install Terraform and Terraform plugins
-RUN wget https://releases.hashicorp.com/terraform/1.1.3/terraform_1.1.3_linux_amd64.zip
-RUN unzip terraform_1.1.3_linux_amd64.zip && rm terraform_1.1.3_linux_amd64.zip
-RUN mv terraform /usr/bin/terraform
-
-RUN wget https://releases.hashicorp.com/terraform-provider-azurerm/2.95.0/terraform-provider-azurerm_2.95.0_linux_amd64.zip
-RUN wget https://releases.hashicorp.com/terraform-provider-aws/3.72.0/terraform-provider-aws_3.72.0_linux_amd64.zip
-RUN wget https://releases.hashicorp.com/terraform-provider-google/4.10.0/terraform-provider-google_4.10.0_linux_amd64.zip
-RUN unzip terraform-provider-azurerm_2.95.0_linux_amd64.zip && rm terraform-provider-azurerm_2.95.0_linux_amd64.zip
-RUN unzip terraform-provider-google_4.10.0_linux_amd64.zip && rm terraform-provider-google_4.10.0_linux_amd64.zip
-RUN unzip terraform-provider-aws_3.72.0_linux_amd64.zip && rm terraform-provider-aws_3.72.0_linux_amd64.zip
-RUN mkdir ~/.terraform.d && mkdir ~/.terraform.d/plugins && mkdir ~/.terraform.d/plugins/linux_amd64 && mv terraform-provider-aws_v3.72.0_x5 terraform-provider-google_v4.10.0_x5 terraform-provider-azurerm_v2.95.0_x5 ~/.terraform.d/plugins/linux_amd64
-
-USER ${KUSER}
-
 # Copy built binary to the runtime container
 COPY --chown=${KUSER}:${KGROUP} --from=build_env /build/bin/kics /app/bin/kics
 COPY --chown=${KUSER}:${KGROUP} --from=build_env /build/assets/ /app/bin/assets/

docs/integrations_terraformer.md

@@ -1,4 +1,4 @@
-# Running KICS with Terraformer
+# Running KICS with Terraformer (Deprecated after 1.7.13)
 
 From version 1.5, KICS integrates with Terraformer to scan resources deployed in the Cloud. The runtime information of the resources is obtained by providing a Terraformer path to KICS, via `-p` flag. The scan happens immediately after this information is obtained. In the end, results are shown as for any other KICS scan.
 

docs/release_checklist.md

@@ -20,4 +20,3 @@
     - git checkout integrations
     - git tag ${latest_tag}-integrations
     - git push origin ${latest_tag}-integrations
-    NOTE: make sure that `pkg/terraformer` only has `terraformer_alt.go` without build tags

pkg/engine/provider/extract.go

@@ -13,7 +13,6 @@ import (
 
 	"github.com/Checkmarx/kics/pkg/kuberneter"
 	"github.com/Checkmarx/kics/pkg/model"
-	"github.com/Checkmarx/kics/pkg/terraformer"
 	"github.com/Checkmarx/kics/pkg/utils"
 	"github.com/rs/zerolog/log"
 
@@ -44,32 +43,6 @@ type getterStruct struct {
 	source      string
 }
 
-// GetTerraformerSources uses Terraformer to download runtime resources from AWS provider
-// to terraform.
-// After Downloaded files kics scan the files as normal local files
-func GetTerraformerSources(source []string, destinationPath string) (ExtractedPath, error) {
-	extrStruct := ExtractedPath{
-		Path:          []string{},
-		ExtractionMap: make(map[string]model.ExtractedPathObject),
-	}
-
-	for _, path := range source {
-		exportedPath, err := terraformer.Import(path, destinationPath)
-		if err != nil {
-			log.Error().Msgf("failed to import %s: %s", path, err)
-		}
-
-		extrStruct.ExtractionMap[exportedPath] = model.ExtractedPathObject{
-			Path:      exportedPath,
-			LocalPath: true,
-		}
-
-		extrStruct.Path = append(extrStruct.Path, exportedPath)
-	}
-
-	return extrStruct, nil
-}
-
 // GetKuberneterSources uses Kubernetes API to download runtime resources
 // After Downloaded files kics scan the files as normal local files
 func GetKuberneterSources(ctx context.Context, source []string, destinationPath string) (ExtractedPath, error) {

pkg/scan/utils.go

@@ -19,8 +19,7 @@ import (
 )
 
 var (
-	terraformerRegex = regexp.MustCompile(`^terraformer::`)
-	kuberneterRegex  = regexp.MustCompile(`^kuberneter::`)
+	kuberneterRegex = regexp.MustCompile(`^kuberneter::`)
 )
 
 func (c *Client) prepareAndAnalyzePaths(ctx context.Context) (provider.ExtractedPath, error) {
@@ -29,12 +28,7 @@ func (c *Client) prepareAndAnalyzePaths(ctx context.Context) (provider.Extracted
 		return provider.ExtractedPath{}, err
 	}
 
-	regularPaths, terraformerPaths, kuberneterPaths := extractPathType(c.ScanParams.Path)
-
-	terraformerExPaths, err := provider.GetTerraformerSources(terraformerPaths, c.ScanParams.OutputPath)
-	if err != nil {
-		return provider.ExtractedPath{}, err
-	}
+	regularPaths, kuberneterPaths := extractPathType(c.ScanParams.Path)
 
 	kuberneterExPaths, err := provider.GetKuberneterSources(ctx, kuberneterPaths, c.ScanParams.OutputPath)
 	if err != nil {
@@ -46,7 +40,7 @@ func (c *Client) prepareAndAnalyzePaths(ctx context.Context) (provider.Extracted
 		return provider.ExtractedPath{}, err
 	}
 
-	allPaths := combinePaths(terraformerExPaths, kuberneterExPaths, regularExPaths, queryExPaths, libExPaths)
+	allPaths := combinePaths(kuberneterExPaths, regularExPaths, queryExPaths, libExPaths)
 	if len(allPaths.Path) == 0 {
 		return provider.ExtractedPath{}, nil
 	}
@@ -79,20 +73,16 @@ func (c *Client) prepareAndAnalyzePaths(ctx context.Context) (provider.Extracted
 	return allPaths, nil
 }
 
-func combinePaths(terraformer, kuberneter, regular, query, library provider.ExtractedPath) provider.ExtractedPath {
+func combinePaths(kuberneter, regular, query, library provider.ExtractedPath) provider.ExtractedPath {
 	var combinedPaths provider.ExtractedPath
 	paths := make([]string, 0)
 	combinedPathsEx := make(map[string]model.ExtractedPathObject)
-	paths = append(paths, terraformer.Path...)
 	paths = append(paths, kuberneter.Path...)
 	paths = append(paths, regular.Path...)
 	combinedPaths.Path = paths
 	for k, v := range regular.ExtractionMap {
 		combinedPathsEx[k] = v
 	}
-	for k, v := range terraformer.ExtractionMap {
-		combinedPathsEx[k] = v
-	}
 	for k, v := range kuberneter.ExtractionMap {
 		combinedPathsEx[k] = v
 	}
@@ -209,11 +199,9 @@ func logLoadingQueriesType(types []string) {
 	log.Info().Msgf("Loading queries of type: %s", strings.Join(types, ", "))
 }
 
-func extractPathType(paths []string) (regular, terraformer, kuberneter []string) {
+func extractPathType(paths []string) (regular, kuberneter []string) {
 	for _, path := range paths {
-		if terraformerRegex.MatchString(path) {
-			terraformer = append(terraformer, terraformerRegex.ReplaceAllString(path, ""))
-		} else if kuberneterRegex.MatchString(path) {
+		if kuberneterRegex.MatchString(path) {
 			kuberneter = append(kuberneter, kuberneterRegex.ReplaceAllString(path, ""))
 		} else {
 			regular = append(regular, path)
@@ -224,8 +212,7 @@ func extractPathType(paths []string) (regular, terraformer, kuberneter []string)
 
 func deleteExtractionFolder(extractionMap map[string]model.ExtractedPathObject) {
 	for extractionFile := range extractionMap {
-		if strings.Contains(extractionFile, "kics-extract-terraformer") ||
-			strings.Contains(extractionFile, "kics-extract-kuberneter") {
+		if strings.Contains(extractionFile, "kics-extract-kuberneter") {
 			continue
 		}
 		err := os.RemoveAll(extractionFile)