Skip to content
This repository was archived by the owner on Mar 20, 2024. It is now read-only.

multiple device request issue #81

Merged
merged 5 commits into from
Nov 28, 2023
Merged

multiple device request issue #81

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d717535
multiple device request issue
maryamtahhan Nov 7, 2023
8e4da0c
device-plugin: fix container env var setting for multiple devices
maryamtahhan Nov 14, 2023
6ec1b90
bpfmap path: fixup the bpf map path to match the sock path
maryamtahhan Nov 14, 2023
d3106ed
multiple-pools: fixup test cases
maryamtahhan Nov 15, 2023
1ea1ad4
constants: uds pod path at /tmp/afxdp_dp
maryamtahhan Nov 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,14 +73,14 @@ build: builddp buildcni
docker: ## Build docker image
@echo "****** Docker Image ******"
@echo
docker build -t localhost:5000/afxdp-device-plugin -f images/amd64.dockerfile .
docker build -t afxdp-device-plugin -f images/amd64.dockerfile .
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@patrickog11 - can you remind me why we had "localhost:5000" here?
Is it needed? Was it just a workaround on our side?

@echo
@echo

podman: ## Build podman image
@echo "****** Podman Image ******"
@echo
podman build -t localhost:5000/afxdp-device-plugin -f images/amd64.dockerfile .
podman build -t afxdp-device-plugin -f images/amd64.dockerfile .
@echo
@echo

Expand Down
23 changes: 13 additions & 10 deletions constants/constants.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ var (

/* Devices */
devicesProhibited = []string{"eno", "eth", "lo", "docker", "flannel", "cni"} // interfaces we never add to a pool
devicesEnvVar = "AFXDP_DEVICES" // env var set in the end user application pod, lists AF_XDP devices attached
devicesEnvVarPrefix = "AFXDP_DEVICES_" // env var set in the end user application pod, lists AF_XDP devices attached
deviceValidNameRegex = `^[a-zA-Z0-9_-]+$` // regex to check if a string is a valid device name
deviceValidNameMin = 1 // minimum length of a device name
deviceValidNameMax = 50 // maximum length of a device name
Expand Down Expand Up @@ -74,16 +74,17 @@ var (
afxdpMinimumLinux = "4.18.0" // minimum Linux version for AF_XDP support

/* UDS*/
udsMaxTimeout = 300 // maximum configurable uds timeout in seconds
udsMinTimeout = 30 // minimum (and default) uds timeout in seconds
udsMsgBufSize = 64 // uds message buffer size
udsCtlBufSize = 4 // uds control buffer size
udsProtocol = "unixpacket" // uds protocol: "unix"=SOCK_STREAM, "unixdomain"=SOCK_DGRAM, "unixpacket"=SOCK_SEQPACKET
udsSockDir = "/tmp/afxdp_dp/" // host location where we place our uds sockets. If changing location remember to update daemonset mount point
udsPodPath = "/tmp/afxdp.sock" // the uds filepath as it will appear in the end user application pod
udsMaxTimeout = 300 // maximum configurable uds timeout in seconds
udsMinTimeout = 30 // minimum (and default) uds timeout in seconds
udsMsgBufSize = 64 // uds message buffer size
udsCtlBufSize = 4 // uds control buffer size
udsProtocol = "unixpacket" // uds protocol: "unix"=SOCK_STREAM, "unixdomain"=SOCK_DGRAM, "unixpacket"=SOCK_SEQPACKET
udsSockDir = "/tmp/afxdp_dp/" // host location where we place our uds sockets. If changing location remember to update daemonset mount point
udsPodPath = "/tmp/afxdp_dp/" // the uds filepath as it will appear in the end user application pod
udsPodSock = "/afxdp.sock"

/* BPF*/
bpfMapPodPath = "/tmp/xsks_map"
bpfMapPodPath = "/tmp/afxdp_dp/"
xsk_map = "/xsks_map"

udsDirFileMode = 0700 // permissions for the directory in which we create our uds sockets
Expand Down Expand Up @@ -216,6 +217,7 @@ type uds struct {
SockDir string
DirFileMode int
PodPath string
SockName string
Handshake handshake
}

Expand Down Expand Up @@ -284,7 +286,7 @@ func init() {

Devices = devices{
Prohibited: devicesProhibited,
EnvVarList: devicesEnvVar,
EnvVarList: devicesEnvVarPrefix,
ValidNameRegex: deviceValidNameRegex,
ValidNameMin: deviceValidNameMin,
ValidNameMax: deviceValidNameMax,
Expand Down Expand Up @@ -326,6 +328,7 @@ func init() {
SockDir: udsSockDir,
DirFileMode: udsDirFileMode,
PodPath: udsPodPath,
SockName: udsPodSock,
Handshake: handshake{
Version: handshakeHandshakeVersion,
RequestVersion: handshakeRequestVersion,
Expand Down
27 changes: 15 additions & 12 deletions internal/deviceplugin/poolManager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,20 +196,22 @@ func (pm *PoolManager) Allocate(ctx context.Context,
cresp := new(pluginapi.ContainerAllocateResponse)
envs := make(map[string]string)

if !pm.UdsServerDisable {
cresp.Mounts = append(cresp.Mounts, &pluginapi.Mount{
HostPath: udsPath,
ContainerPath: constants.Uds.PodPath,
ReadOnly: false,
})
}

//loop each device request per container
for _, devName := range crqt.DevicesIDs {
device := pm.Devices[devName]
pretty, _ := tools.PrettyString(device.Public())
logging.Debugf("Device: %s", pretty)

containerSockPath := constants.Uds.PodPath + device.Name() + constants.Uds.SockName

if !pm.UdsServerDisable {
cresp.Mounts = append(cresp.Mounts, &pluginapi.Mount{
HostPath: udsPath,
ContainerPath: containerSockPath,
ReadOnly: false,
})
}

if device.Mode() != pm.Mode {
err := fmt.Errorf("pool mode %s does not match device mode %s", pm.Mode, device.Mode())
logging.Errorf("%v", err)
Expand Down Expand Up @@ -265,16 +267,18 @@ func (pm *PoolManager) Allocate(ctx context.Context,

//FULL PATH WILL INCLUDE THE XSKMAP...
fullPath := pinPath + constants.Bpf.Xsk_map
logging.Debugf("mapping %s to %s", fullPath, constants.Bpf.BpfMapPodPath)
containerMapPath := constants.Bpf.BpfMapPodPath + device.Name() + constants.Bpf.Xsk_map
logging.Debugf("mapping %s to %s", fullPath, containerMapPath)
cresp.Mounts = append(cresp.Mounts, &pluginapi.Mount{
HostPath: fullPath,
ContainerPath: constants.Bpf.BpfMapPodPath,
ContainerPath: containerMapPath,
ReadOnly: false,
})
}
}

envs[constants.Devices.EnvVarList] = strings.Join(crqt.DevicesIDs, " ")
envVar := constants.Devices.EnvVarList + strings.ToUpper(pm.Name)
envs[envVar] = strings.Join(crqt.DevicesIDs, " ")
envsPrint, err := tools.PrettyString(envs)
if err != nil {
logging.Errorf("Error printing container environment variables: %v", err)
Expand All @@ -283,7 +287,6 @@ func (pm *PoolManager) Allocate(ctx context.Context,
}
cresp.Envs = envs
response.ContainerResponses = append(response.ContainerResponses, cresp)

}

if !pm.UdsServerDisable {
Expand Down
67 changes: 47 additions & 20 deletions internal/deviceplugin/poolManager_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package deviceplugin
import (
"context"
"encoding/json"
"strings"
"testing"

"github.com/intel/afxdp-plugins-for-kubernetes/constants"
Expand Down Expand Up @@ -57,6 +58,8 @@ func TestAllocate(t *testing.T) {
pm.ServerFactory = udsserver.NewFakeServerFactory()
pm.BpfHandler = bpf.NewFakeHandler()

envVar := constants.Devices.EnvVarList + strings.ToUpper(pm.Name)

testCases := []struct {
name string
containerRequests []*pluginapi.ContainerAllocateRequest
Expand All @@ -69,10 +72,10 @@ func TestAllocate(t *testing.T) {
},
expContainerResponses: []*pluginapi.ContainerAllocateResponse{
{
Envs: map[string]string{constants.Devices.EnvVarList: "dev_1"},
Envs: map[string]string{envVar: "dev_1"},
Mounts: []*pluginapi.Mount{
{
ContainerPath: constants.Uds.PodPath,
ContainerPath: constants.Uds.PodPath + "dev_1" + constants.Uds.SockName,
HostPath: "/tmp/fake-socket.sock",
ReadOnly: false,
},
Expand All @@ -90,10 +93,20 @@ func TestAllocate(t *testing.T) {
},
expContainerResponses: []*pluginapi.ContainerAllocateResponse{
{
Envs: map[string]string{constants.Devices.EnvVarList: "dev_1 dev_2 dev_3"},
Envs: map[string]string{envVar: "dev_1 dev_2 dev_3"},
Mounts: []*pluginapi.Mount{
{
ContainerPath: constants.Uds.PodPath,
ContainerPath: constants.Uds.PodPath + "dev_1" + constants.Uds.SockName,
HostPath: "/tmp/fake-socket.sock",
ReadOnly: false,
},
{
ContainerPath: constants.Uds.PodPath + "dev_2" + constants.Uds.SockName,
HostPath: "/tmp/fake-socket.sock",
ReadOnly: false,
},
{
ContainerPath: constants.Uds.PodPath + "dev_3" + constants.Uds.SockName,
HostPath: "/tmp/fake-socket.sock",
ReadOnly: false,
},
Expand All @@ -112,10 +125,10 @@ func TestAllocate(t *testing.T) {
},
expContainerResponses: []*pluginapi.ContainerAllocateResponse{
{
Envs: map[string]string{constants.Devices.EnvVarList: "dev_1"},
Envs: map[string]string{envVar: "dev_1"},
Mounts: []*pluginapi.Mount{
{
ContainerPath: constants.Uds.PodPath,
ContainerPath: constants.Uds.PodPath + "dev_1" + constants.Uds.SockName,
HostPath: "/tmp/fake-socket.sock",
ReadOnly: false,
},
Expand All @@ -124,10 +137,10 @@ func TestAllocate(t *testing.T) {
Annotations: map[string]string{},
},
{
Envs: map[string]string{constants.Devices.EnvVarList: "dev_2"},
Envs: map[string]string{envVar: "dev_2"},
Mounts: []*pluginapi.Mount{
{
ContainerPath: constants.Uds.PodPath,
ContainerPath: constants.Uds.PodPath + "dev_2" + constants.Uds.SockName,
HostPath: "/tmp/fake-socket.sock",
ReadOnly: false,
},
Expand All @@ -146,10 +159,20 @@ func TestAllocate(t *testing.T) {
},
expContainerResponses: []*pluginapi.ContainerAllocateResponse{
{
Envs: map[string]string{constants.Devices.EnvVarList: "dev_1 dev_2 dev_3"},
Envs: map[string]string{envVar: "dev_1 dev_2 dev_3"},
Mounts: []*pluginapi.Mount{
{
ContainerPath: constants.Uds.PodPath,
ContainerPath: constants.Uds.PodPath + "dev_1" + constants.Uds.SockName,
HostPath: "/tmp/fake-socket.sock",
ReadOnly: false,
},
{
ContainerPath: constants.Uds.PodPath + "dev_2" + constants.Uds.SockName,
HostPath: "/tmp/fake-socket.sock",
ReadOnly: false,
},
{
ContainerPath: constants.Uds.PodPath + "dev_3" + constants.Uds.SockName,
HostPath: "/tmp/fake-socket.sock",
ReadOnly: false,
},
Expand All @@ -158,10 +181,20 @@ func TestAllocate(t *testing.T) {
Annotations: map[string]string{},
},
{
Envs: map[string]string{constants.Devices.EnvVarList: "dev_4 dev_5 dev_6"},
Envs: map[string]string{envVar: "dev_4 dev_5 dev_6"},
Mounts: []*pluginapi.Mount{
{
ContainerPath: constants.Uds.PodPath,
ContainerPath: constants.Uds.PodPath + "dev_4" + constants.Uds.SockName,
HostPath: "/tmp/fake-socket.sock",
ReadOnly: false,
},
{
ContainerPath: constants.Uds.PodPath + "dev_5" + constants.Uds.SockName,
HostPath: "/tmp/fake-socket.sock",
ReadOnly: false,
},
{
ContainerPath: constants.Uds.PodPath + "dev_6" + constants.Uds.SockName,
HostPath: "/tmp/fake-socket.sock",
ReadOnly: false,
},
Expand All @@ -179,14 +212,8 @@ func TestAllocate(t *testing.T) {
},
expContainerResponses: []*pluginapi.ContainerAllocateResponse{
{
Envs: map[string]string{constants.Devices.EnvVarList: ""},
Mounts: []*pluginapi.Mount{
{
ContainerPath: constants.Uds.PodPath,
HostPath: "/tmp/fake-socket.sock",
ReadOnly: false,
},
},
Envs: map[string]string{envVar: ""},
Mounts: []*pluginapi.Mount{},
Devices: []*pluginapi.DeviceSpec{},
Annotations: map[string]string{},
},
Expand Down