Skip to content

Commit 53c2ae6

Browse files
authored
Separate sandbox traffic to avoid capturing host packets during analysis. (#275)
- use a virtual bridge to separate sandbox traffic from host traffic. - switch from pcap to pcapgo to avoid a deadlock (afpacket would be nice, but it segfaults) - explicitly define the podman network so pcap can be started before podman - `Close()` now explicitly aborts the packet reading.
1 parent d0506c8 commit 53c2ae6

File tree

9 files changed

+219
-92
lines changed

9 files changed

+219
-92
lines changed

cmd/analyze/Dockerfile

+5-3
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ RUN apt-get update && apt-get upgrade -y && \
1313
ca-certificates \
1414
curl \
1515
iptables \
16-
libpcap0.8 \
16+
iproute2 \
1717
podman \
1818
software-properties-common && \
1919
update-alternatives --set iptables /usr/sbin/iptables-legacy && \
@@ -27,8 +27,10 @@ RUN curl -fsSL https://gvisor.dev/archive.key | apt-key add - && \
2727
COPY --from=build /src/analyze /usr/local/bin/analyze
2828
COPY --from=build /src/worker /usr/local/bin/worker
2929
COPY --from=build /src/tools/gvisor/runsc_compat.sh /usr/local/bin/runsc_compat.sh
30-
COPY --from=build /src/tools/firewall/iptables.rules /usr/local/etc/iptables.rules
31-
RUN chmod 755 /usr/local/bin/runsc_compat.sh
30+
COPY --from=build /src/tools/network/iptables.rules /usr/local/etc/iptables.rules
31+
COPY --from=build /src/tools/network/podman-analysis.conflist /etc/cni/net.d/podman-analysis.conflist
32+
RUN chmod 755 /usr/local/bin/runsc_compat.sh && \
33+
chmod 644 /usr/local/etc/iptables.rules /etc/cni/net.d/podman-analysis.conflist
3234

3335
ARG SANDBOX_IMAGE_TAG
3436
ENV OSSF_SANDBOX_IMAGE_TAG=${SANDBOX_IMAGE_TAG}

cmd/analyze/main.go

+2
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ func parseBucketPath(path string) (string, string) {
3535

3636
func main() {
3737
log.Initalize(os.Getenv("LOGGER_ENV"))
38+
sandbox.InitEnv()
39+
3840
flag.Parse()
3941
if *ecosystem == "" {
4042
flag.Usage()

cmd/worker/main.go

+1
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,7 @@ func main() {
229229
resultsBucket := os.Getenv("OSSF_MALWARE_ANALYSIS_RESULTS")
230230
imageTag := os.Getenv("OSSF_SANDBOX_IMAGE_TAG")
231231
log.Initalize(os.Getenv("LOGGER_ENV"))
232+
sandbox.InitEnv()
232233

233234
// Log the configuration of the worker at startup so we can observe it.
234235
log.Info("Starting worker",

internal/analysis/analysis.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ func Run(sb sandbox.Sandbox, args []string) (*Result, error) {
102102
"args", args)
103103

104104
log.Debug("Preparing packet capture")
105-
pcap := packetcapture.New()
105+
pcap := packetcapture.New(sandbox.NetworkInterface)
106106

107107
dns := dnsanalyzer.New()
108108
pcap.RegisterReceiver(dns)
@@ -119,6 +119,7 @@ func Run(sb sandbox.Sandbox, args []string) (*Result, error) {
119119
return resultError, fmt.Errorf("sandbox failed (%w)", err)
120120
}
121121

122+
log.Debug("Stop the packet capture")
122123
pcap.Close()
123124

124125
// Grab the log file

internal/packetcapture/packetcapture.go

+32-27
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,8 @@ package packetcapture
22

33
import (
44
"github.com/google/gopacket"
5-
"github.com/google/gopacket/pcap"
6-
)
7-
8-
const (
9-
captureDevice = "eth0"
5+
"github.com/google/gopacket/layers"
6+
"github.com/google/gopacket/pcapgo"
107
)
118

129
// PacketReceiver implementations can be registered with a PacketCapture to be
@@ -20,18 +17,22 @@ type PacketReceiver interface {
2017
type Handler func(gopacket.Layer, gopacket.Packet)
2118

2219
type PacketCapture struct {
23-
handle *pcap.Handle
20+
netInterface string
21+
handle *pcapgo.EthernetHandle
2422
done chan bool
23+
stop chan bool
2524
packetReceivers map[gopacket.LayerType][]PacketReceiver
2625
}
2726

28-
// New returns a new Trace instance.
27+
// New returns a new PacketCapture instance for the given netInterface
2928
//
30-
// Close() must be called on the Trace instance.
31-
func New() *PacketCapture {
29+
// Close() must be called on the PacketCapture instance.
30+
func New(netInterface string) *PacketCapture {
3231
return &PacketCapture{
32+
netInterface: netInterface,
3333
handle: nil,
3434
done: make(chan bool),
35+
stop: make(chan bool),
3536
packetReceivers: make(map[gopacket.LayerType][]PacketReceiver),
3637
}
3738
}
@@ -53,34 +54,38 @@ func (pc *PacketCapture) RegisterReceiver(receiver PacketReceiver) {
5354
}
5455

5556
func (pc *PacketCapture) Start() error {
56-
inactive, err := pcap.NewInactiveHandle(captureDevice)
57-
if err != nil {
58-
return err
59-
}
60-
defer inactive.CleanUp()
61-
62-
// Force packets to be sent immediately to ensure we don't miss any.
63-
if err := inactive.SetImmediateMode(true); err != nil {
64-
return err
65-
}
66-
67-
handle, err := inactive.Activate()
57+
// Use the pcapgo library for capturing traffic as it is the most reliable.
58+
// afpacket is the fastest but segfaults: https://github.com/google/gopacket/issues/717
59+
// pcap will block during Cancel(): https://github.com/google/gopacket/issues/890
60+
handle, err := pcapgo.NewEthernetHandle(pc.netInterface)
6861
if err != nil {
6962
return err
7063
}
7164
pc.handle = handle
72-
packetSource := gopacket.NewPacketSource(pc.handle, pc.handle.LinkType())
73-
go func(packets chan gopacket.Packet, done chan bool) {
74-
for packet := range packets {
75-
pc.handlePacket(packet)
65+
packetSource := gopacket.NewPacketSource(pc.handle, layers.LinkTypeEthernet)
66+
go func(packets chan gopacket.Packet, done chan bool, stop chan bool) {
67+
defer func() { done <- true }()
68+
for {
69+
select {
70+
case packet, ok := <-packets:
71+
if ok {
72+
pc.handlePacket(packet)
73+
} else {
74+
return
75+
}
76+
case v, ok := <-stop:
77+
if !ok || v {
78+
return
79+
}
80+
}
7681
}
77-
done <- true
78-
}(packetSource.Packets(), pc.done)
82+
}(packetSource.Packets(), pc.done, pc.stop)
7983
return nil
8084
}
8185

8286
func (pc *PacketCapture) Close() {
8387
if pc.handle != nil {
88+
pc.stop <- true
8489
pc.handle.Close()
8590
// Wait for packet processing to be finished.
8691
<-pc.done

internal/sandbox/init.go

+130
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
package sandbox
2+
3+
import (
4+
"fmt"
5+
"io"
6+
"os"
7+
"os/exec"
8+
9+
"github.com/ossf/package-analysis/internal/log"
10+
)
11+
12+
const (
13+
ipBin = "/usr/sbin/ip"
14+
iptablesLoadBin = "/usr/sbin/iptables-restore"
15+
iptablesRules = "/usr/local/etc/iptables.rules"
16+
dummyInterface = "cnidummy0"
17+
18+
// bridgeInterface is the name of the podman bridge defined in
19+
// tools/network/podman-analysis.conflist. This bridge is used by the
20+
// sandbox during analysis to separate the sandbox traffic from the host.
21+
bridgeInterface = "cni-analysis"
22+
)
23+
24+
const (
25+
// NetworkInterface is the name of a network interface that has access to
26+
// the sandbox network traffic.
27+
NetworkInterface = bridgeInterface
28+
)
29+
30+
func loadIptablesRules() error {
31+
log.Debug("Loading iptable rules")
32+
33+
// Open the iptables-restore configuration
34+
f, err := os.Open(iptablesRules)
35+
if err != nil {
36+
return err
37+
}
38+
defer f.Close()
39+
40+
logOut := log.Writer(log.InfoLevel)
41+
defer logOut.Close()
42+
logErr := log.Writer(log.WarnLevel)
43+
defer logErr.Close()
44+
45+
cmd := exec.Command(iptablesLoadBin)
46+
cmd.Stdout = logOut
47+
cmd.Stderr = logErr
48+
stdin, err := cmd.StdinPipe()
49+
if err != nil {
50+
return err
51+
}
52+
defer stdin.Close()
53+
if err := cmd.Start(); err != nil {
54+
return err
55+
}
56+
// Send the iptables rules to the command via stdin
57+
if _, err := io.Copy(stdin, f); err != nil {
58+
return err
59+
}
60+
stdin.Close()
61+
return cmd.Wait()
62+
}
63+
64+
// createBridgeNetwork ensures that NetworkInterface and the bridge network
65+
// exists prior to the sandbox.
66+
//
67+
// podman would create this bridge interface anyway, but doing it early allows
68+
// a packet capture to be started on the interface prior to the sandbox
69+
// starting.
70+
func createBridgeNetwork() error {
71+
log.Debug("Creating bridge network")
72+
73+
// Create the bridge
74+
cmd := exec.Command(ipBin, "link", "add", "name", bridgeInterface, "type", "bridge")
75+
if err := cmd.Run(); err != nil {
76+
// If the error is not an ExitError, or the Exit Code is not 2, then abort.
77+
if exitErr, ok := err.(*exec.ExitError); !ok || exitErr.ExitCode() != 2 {
78+
return fmt.Errorf("failed to add bridge interface: %w", err)
79+
}
80+
}
81+
82+
// Bring the bridge up.
83+
cmd = exec.Command(ipBin, "link", "set", bridgeInterface, "up")
84+
if err := cmd.Run(); err != nil {
85+
return fmt.Errorf("failed to bring up bridge interface: %w", err)
86+
}
87+
88+
// Add a dummy device so the bridge stays up
89+
cmd = exec.Command(ipBin, "link", "add", "dev", dummyInterface, "type", "dummy")
90+
if err := cmd.Run(); err != nil {
91+
// If the error is not an ExitError, or the Exit Code is not 2, then abort.
92+
if exitErr, ok := err.(*exec.ExitError); !ok || exitErr.ExitCode() != 2 {
93+
return fmt.Errorf("failed to create dummy inteface: %w", err)
94+
}
95+
}
96+
97+
// Add the dummy device to the bridge network
98+
cmd = exec.Command(ipBin, "link", "set", "dev", dummyInterface, "master", bridgeInterface)
99+
if err := cmd.Run(); err != nil {
100+
return fmt.Errorf("failed to add dummy interface to bridge: %w", err)
101+
}
102+
103+
// Bring the dummy device up.
104+
cmd = exec.Command(ipBin, "link", "set", dummyInterface, "up")
105+
if err := cmd.Run(); err != nil {
106+
return fmt.Errorf("failed to bring up dummy interface: %w", err)
107+
}
108+
109+
return nil
110+
}
111+
112+
// InitEnv initializes the host for running sandboxes.
113+
//
114+
// It will ensure that the network interface exists, and any firewall
115+
// rules are configured.
116+
//
117+
// This function is idempotent and is safe to be called more than once.
118+
//
119+
// This function must be called after logging is complete, and may exit if
120+
// any of the commands fail.
121+
func InitEnv() {
122+
// Create the bridge network
123+
if err := createBridgeNetwork(); err != nil {
124+
log.Fatal("Failed to create bridge network", "error", err)
125+
}
126+
// Load iptables rules to further isolate the sandbox
127+
if err := loadIptablesRules(); err != nil {
128+
log.Fatal("Failed restoring iptables rules", "error", err)
129+
}
130+
}

internal/sandbox/sandbox.go

+12-57
Original file line numberDiff line numberDiff line change
@@ -9,19 +9,22 @@ import (
99
"path"
1010
"path/filepath"
1111
"strings"
12-
"text/template"
1312

1413
"github.com/ossf/package-analysis/internal/log"
1514
)
1615

1716
const (
18-
podmanBin = "podman"
19-
runtimeBin = "/usr/local/bin/runsc_compat.sh"
20-
iptablesLoadBin = "/usr/sbin/iptables-restore"
21-
iptablesRules = "/usr/local/etc/iptables.rules"
22-
rootDir = "/var/run/runsc"
23-
straceFile = "runsc.log.boot"
24-
logDirPattern = "sandbox_logs_"
17+
podmanBin = "podman"
18+
runtimeBin = "/usr/local/bin/runsc_compat.sh"
19+
rootDir = "/var/run/runsc"
20+
straceFile = "runsc.log.boot"
21+
logDirPattern = "sandbox_logs_"
22+
23+
// networkName is the name of the podman network defined in
24+
// tools/network/podman-analysis.conflist. This network is the network
25+
// used by the sandbox during analysis to separate the sandbox traffic
26+
// from the host.
27+
networkName = "analysis-net"
2528
)
2629

2730
type RunStatus uint8
@@ -166,37 +169,6 @@ func removeAllLogs() error {
166169
return nil
167170
}
168171

169-
func loadIptablesRules() error {
170-
// Get the subnet for the default podman network
171-
subnet, err := podmanNetworkSubnet()
172-
if err != nil {
173-
return err
174-
}
175-
// Prepare the iptables-restore template
176-
t := template.Must(template.ParseFiles(iptablesRules))
177-
178-
logOut := log.Writer(log.InfoLevel)
179-
defer logOut.Close()
180-
logErr := log.Writer(log.WarnLevel)
181-
defer logErr.Close()
182-
183-
cmd := exec.Command(iptablesLoadBin)
184-
cmd.Stdout = logOut
185-
cmd.Stderr = logErr
186-
stdin, err := cmd.StdinPipe()
187-
if err != nil {
188-
return err
189-
}
190-
defer stdin.Close()
191-
if err := cmd.Start(); err != nil {
192-
return err
193-
}
194-
// Send the iptables rules to the command via stdin
195-
t.Execute(stdin, struct{ Subnet string }{Subnet: subnet})
196-
stdin.Close()
197-
return cmd.Wait()
198-
}
199-
200172
func podman(args ...string) *exec.Cmd {
201173
args = append([]string{
202174
"--cgroup-manager=cgroupfs",
@@ -219,20 +191,6 @@ func podmanCleanContainers() error {
219191
return podmanRun("rm", "--all", "--force")
220192
}
221193

222-
// podmanNetworkSubnet returns the IPv4 subnet of the default "podman" network.
223-
func podmanNetworkSubnet() (string, error) {
224-
args := []string{
225-
"network", "inspect", "podman", "-f", "{{range .plugins}}{{with .ipam.subnet}}{{.}}{{end}}{{end}}",
226-
}
227-
cmd := podman(args...)
228-
var buf bytes.Buffer
229-
cmd.Stdout = &buf
230-
if err := cmd.Run(); err != nil {
231-
return "", err
232-
}
233-
return string(bytes.TrimSpace(buf.Bytes())), nil
234-
}
235-
236194
func (s *podmanSandbox) pullImage() error {
237195
return podmanRun("pull", s.imageWithTag())
238196
}
@@ -245,6 +203,7 @@ func (s *podmanSandbox) createContainer() (string, error) {
245203
"--dns=8.8.8.8", // Manually specify DNS to bypass kube-dns and
246204
"--dns=8.8.4.4", // allow for tighter firewall rules that block
247205
"--dns-search=.", // network traffic to private IP address ranges.
206+
"--network=" + networkName,
248207
}
249208
args = append(args, s.extraArgs()...)
250209
args = append(args, s.imageWithTag())
@@ -311,10 +270,6 @@ func (s *podmanSandbox) init() error {
311270
if s.container != "" {
312271
return nil
313272
}
314-
// Load iptables rules to further isolate the sandbox
315-
if err := loadIptablesRules(); err != nil {
316-
return fmt.Errorf("failed restoring iptables rules: %w", err)
317-
}
318273
// Delete existing logs (if any).
319274
if err := removeAllLogs(); err != nil {
320275
return fmt.Errorf("failed removing all logs: %w", err)

0 commit comments

Comments
 (0)