Adds the crates.io ecosystem (#366)

* Add crates.io ecosystem

* Improve docker file for go

* Update build_docker.sh script

* Update crates.io naming

* Rename cratesio to crates

* Actually rename cratesio to crates.

Co-authored-by: Rex P
Co-authored-by: Caleb Brown <[email protected]>

Author: another-rex

build/build_docker.sh

@@ -7,7 +7,7 @@ BASE_PATH="$(dirname $(dirname $(realpath $0)))"
 REGISTRY=gcr.io/ossf-malware-analysis
 
 # Mapping from the container name to the path containing the Dockerfile.
-declare -A ANALYSIS_IMAGES=( [node]=npm [python]=pypi [ruby]=rubygems [packagist]=packagist )
+declare -A ANALYSIS_IMAGES=( [node]=npm [python]=pypi [ruby]=rubygems [packagist]=packagist [crates.io]=crates.io )
 
 pushd "$BASE_PATH/sandboxes"
 for image in "${!ANALYSIS_IMAGES[@]}"; do

cmd/analyze/Dockerfile

@@ -1,6 +1,12 @@
 FROM golang@sha256:3c4de86eec9cbc619cdd72424abd88326ffcf5d813a8338a7743c55e5898734f as build
 RUN apt-get update && apt-get install -y libpcap-dev
 WORKDIR /src
+
+# First cache the dependencies to avoid downloading again on code change
+COPY ./go.mod ./
+COPY ./go.sum ./
+RUN go mod download
+
 COPY . ./
 RUN go build -o analyze cmd/analyze/main.go && go build -o worker cmd/worker/main.go
 

cmd/scheduler/Dockerfile

@@ -1,5 +1,11 @@
 FROM golang@sha256:3c4de86eec9cbc619cdd72424abd88326ffcf5d813a8338a7743c55e5898734f as build
 WORKDIR /src
+
+# First cache the dependencies to avoid downloading again on code change
+COPY ./go.mod ./
+COPY ./go.sum ./
+RUN go mod download
+
 COPY . ./
 RUN go build -o scheduler ./cmd/scheduler/main.go
 

internal/pkgecosystem/crates.io.go

@@ -0,0 +1,40 @@
+package pkgecosystem
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+)
+
+type cratesJSON struct {
+	Versions []struct {
+		Num string `json:"num"`
+	} `json:"versions"`
+}
+
+func getCratesLatest(pkg string) (string, error) {
+	resp, err := http.Get(fmt.Sprintf("https://crates.io/api/v1/crates/%s/versions", pkg))
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	decoder := json.NewDecoder(resp.Body)
+	var details cratesJSON
+	err = decoder.Decode(&details)
+	if err != nil {
+		return "", err
+	}
+
+	return details.Versions[0].Num, nil
+}
+
+var cratesPkgManager = PkgManager{
+	name:    "crates.io",
+	image:   "gcr.io/ossf-malware-analysis/crates.io",
+	command: "/usr/local/bin/analyze.py",
+	latest:  getCratesLatest,
+	dynamicPhases: []string{
+		"install",
+	},
+}

internal/pkgecosystem/ecosystem.go

@@ -18,6 +18,7 @@ var (
 		pypiPkgManager.name:      &pypiPkgManager,
 		rubygemsPkgManager.name:  &rubygemsPkgManager,
 		packagistPkgManager.name: &packagistPkgManager,
+		cratesPkgManager.name:    &cratesPkgManager,
 	}
 )
 

sandboxes/crates.io/Dockerfile

@@ -0,0 +1,25 @@
+FROM rust:1.63.0@sha256:54f17dc625ed83d3fc7dee68d5eac4e1977de40e72d909bcdb50cc55a462779d as image
+
+RUN apt-get update && \
+    apt-get install -y \
+        curl \
+        wget \
+        git \
+        python
+
+COPY analyze.py /usr/local/bin/
+RUN chmod 755 /usr/local/bin/analyze.py
+RUN mkdir -p /app
+WORKDIR /app
+RUN cargo init
+
+FROM scratch
+COPY --from=image / /
+WORKDIR /app
+ENV PATH="/usr/local/cargo/bin:${PATH}"
+ENV RUSTUP_HOME="/usr/local/rustup"
+ENV CARGO_HOME="/usr/local/cargo"
+
+ENTRYPOINT [ "sleep" ]
+
+CMD [ "30m" ]

sandboxes/crates.io/analyze.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+from dataclasses import dataclass
+import sys
+import subprocess
+from typing import Optional
+
+@dataclass
+class Package:
+    """Class for tracking a package."""
+    name: str
+    version: Optional[str] = None
+    local_path: Optional[str] = None
+
+    def get_dependency_line(self):
+      if self.local_path:
+        return f'{self.name} = {{ path = "{self.local_path}" }}'
+      elif self.version:
+        return f'{self.name} = "{self.version}"'
+      else:
+        return f'{self.name} = "*"'
+
+def install(package: Package):
+    """Cargo build."""
+    try:
+      with open("Cargo.toml", 'a') as handle:
+        handle.write(package.get_dependency_line() + '\n')
+        handle.flush()
+    
+      output = subprocess.check_output(['cargo', 'build'], stderr=subprocess.STDOUT)
+      
+      print('Install succeeded:')
+      print(output.decode())
+    except subprocess.CalledProcessError as e:
+      print('Failed to install:')
+      print(e.output.decode())
+      # Always raise.
+      # Install failing is either an interesting issue, or an opportunity to
+      # improve the analysis.
+      raise
+
+
+PHASES = {
+    "all": [install],
+    "install": [install],
+}
+
+def main():
+    args = list(sys.argv)
+    script = args.pop(0)
+
+    if len(args) < 2 or len(args) > 4:
+        raise ValueError(f'Usage: {script} [--local file | --version version] phase package_name')
+
+    # Parse the arguments manually to avoid introducing unnecessary dependencies
+    # and side effects that add noise to the strace output.
+    local_path = None
+    version = None
+    if args[0] == '--local':
+        args.pop(0)
+        local_path = args.pop(0)
+    elif args[0] == '--version':
+        args.pop(0)
+        version = args.pop(0)
+
+    phase = args.pop(0)
+    package_name = args.pop(0)
+
+    if not phase in PHASES:
+        print(f'Unknown phase {phase} specified.')
+        exit(1)
+
+    package = Package(name=package_name, version=version, local_path=local_path)
+
+    # Execute for the specified phase.
+    for phase in PHASES[phase]:
+        phase(package)
+
+
+if __name__ == '__main__':
+    main()