Skip to content
This repository was archived by the owner on Mar 20, 2024. It is now read-only.

Commit 720d92b

Browse files
maryamtahhanmaryamtahhan
authored
multiple device request issue (#81)
* multiple device request issue the issue is if a single pod requests different devices from different pools it results in multiple uds servers serving the same container and all attempt to mount their uds to the pod as /tmp/afxdp.sock. A similar issue exists for the AFXDP_DEVICES env var that's set in each container. This patch fixes the first issue by mounting the xsksocket at /tmp/afxdp_dp/<netdev>/afxdp.sock, a similar issue also existed for the bpf map pinning support. --------- Signed-off-by: Maryam Tahhan <[email protected]>
1 parent 38317c2 commit 720d92b

File tree

4 files changed

+77
-44
lines changed

4 files changed

+77
-44
lines changed

‎Makefile

+2-2
Original file line numberDiff line numberDiff line change
@@ -73,14 +73,14 @@ build: builddp buildcni
7373
docker: ## Build docker image
7474
@echo "****** Docker Image ******"
7575
@echo
76-
docker build -t localhost:5000/afxdp-device-plugin -f images/amd64.dockerfile .
76+
docker build -t afxdp-device-plugin -f images/amd64.dockerfile .
7777
@echo
7878
@echo
7979

8080
podman: ## Build podman image
8181
@echo "****** Podman Image ******"
8282
@echo
83-
podman build -t localhost:5000/afxdp-device-plugin -f images/amd64.dockerfile .
83+
podman build -t afxdp-device-plugin -f images/amd64.dockerfile .
8484
@echo
8585
@echo
8686

‎constants/constants.go

+13-10
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ var (
4040

4141
/* Devices */
4242
devicesProhibited = []string{"eno", "eth", "lo", "docker", "flannel", "cni"} // interfaces we never add to a pool
43-
devicesEnvVar = "AFXDP_DEVICES" // env var set in the end user application pod, lists AF_XDP devices attached
43+
devicesEnvVarPrefix = "AFXDP_DEVICES_" // env var set in the end user application pod, lists AF_XDP devices attached
4444
deviceValidNameRegex = `^[a-zA-Z0-9_-]+$` // regex to check if a string is a valid device name
4545
deviceValidNameMin = 1 // minimum length of a device name
4646
deviceValidNameMax = 50 // maximum length of a device name
@@ -74,16 +74,17 @@ var (
7474
afxdpMinimumLinux = "4.18.0" // minimum Linux version for AF_XDP support
7575

7676
/* UDS*/
77-
udsMaxTimeout = 300 // maximum configurable uds timeout in seconds
78-
udsMinTimeout = 30 // minimum (and default) uds timeout in seconds
79-
udsMsgBufSize = 64 // uds message buffer size
80-
udsCtlBufSize = 4 // uds control buffer size
81-
udsProtocol = "unixpacket" // uds protocol: "unix"=SOCK_STREAM, "unixdomain"=SOCK_DGRAM, "unixpacket"=SOCK_SEQPACKET
82-
udsSockDir = "/tmp/afxdp_dp/" // host location where we place our uds sockets. If changing location remember to update daemonset mount point
83-
udsPodPath = "/tmp/afxdp.sock" // the uds filepath as it will appear in the end user application pod
77+
udsMaxTimeout = 300 // maximum configurable uds timeout in seconds
78+
udsMinTimeout = 30 // minimum (and default) uds timeout in seconds
79+
udsMsgBufSize = 64 // uds message buffer size
80+
udsCtlBufSize = 4 // uds control buffer size
81+
udsProtocol = "unixpacket" // uds protocol: "unix"=SOCK_STREAM, "unixdomain"=SOCK_DGRAM, "unixpacket"=SOCK_SEQPACKET
82+
udsSockDir = "/tmp/afxdp_dp/" // host location where we place our uds sockets. If changing location remember to update daemonset mount point
83+
udsPodPath = "/tmp/afxdp_dp/" // the uds filepath as it will appear in the end user application pod
84+
udsPodSock = "/afxdp.sock"
8485

8586
/* BPF*/
86-
bpfMapPodPath = "/tmp/xsks_map"
87+
bpfMapPodPath = "/tmp/afxdp_dp/"
8788
xsk_map = "/xsks_map"
8889

8990
udsDirFileMode = 0700 // permissions for the directory in which we create our uds sockets
@@ -216,6 +217,7 @@ type uds struct {
216217
SockDir string
217218
DirFileMode int
218219
PodPath string
220+
SockName string
219221
Handshake handshake
220222
}
221223

@@ -284,7 +286,7 @@ func init() {
284286

285287
Devices = devices{
286288
Prohibited: devicesProhibited,
287-
EnvVarList: devicesEnvVar,
289+
EnvVarList: devicesEnvVarPrefix,
288290
ValidNameRegex: deviceValidNameRegex,
289291
ValidNameMin: deviceValidNameMin,
290292
ValidNameMax: deviceValidNameMax,
@@ -326,6 +328,7 @@ func init() {
326328
SockDir: udsSockDir,
327329
DirFileMode: udsDirFileMode,
328330
PodPath: udsPodPath,
331+
SockName: udsPodSock,
329332
Handshake: handshake{
330333
Version: handshakeHandshakeVersion,
331334
RequestVersion: handshakeRequestVersion,

‎internal/deviceplugin/poolManager.go

+15-12
Original file line numberDiff line numberDiff line change
@@ -196,20 +196,22 @@ func (pm *PoolManager) Allocate(ctx context.Context,
196196
cresp := new(pluginapi.ContainerAllocateResponse)
197197
envs := make(map[string]string)
198198

199-
if !pm.UdsServerDisable {
200-
cresp.Mounts = append(cresp.Mounts, &pluginapi.Mount{
201-
HostPath: udsPath,
202-
ContainerPath: constants.Uds.PodPath,
203-
ReadOnly: false,
204-
})
205-
}
206-
207199
//loop each device request per container
208200
for _, devName := range crqt.DevicesIDs {
209201
device := pm.Devices[devName]
210202
pretty, _ := tools.PrettyString(device.Public())
211203
logging.Debugf("Device: %s", pretty)
212204

205+
containerSockPath := constants.Uds.PodPath + device.Name() + constants.Uds.SockName
206+
207+
if !pm.UdsServerDisable {
208+
cresp.Mounts = append(cresp.Mounts, &pluginapi.Mount{
209+
HostPath: udsPath,
210+
ContainerPath: containerSockPath,
211+
ReadOnly: false,
212+
})
213+
}
214+
213215
if device.Mode() != pm.Mode {
214216
err := fmt.Errorf("pool mode %s does not match device mode %s", pm.Mode, device.Mode())
215217
logging.Errorf("%v", err)
@@ -265,16 +267,18 @@ func (pm *PoolManager) Allocate(ctx context.Context,
265267

266268
//FULL PATH WILL INCLUDE THE XSKMAP...
267269
fullPath := pinPath + constants.Bpf.Xsk_map
268-
logging.Debugf("mapping %s to %s", fullPath, constants.Bpf.BpfMapPodPath)
270+
containerMapPath := constants.Bpf.BpfMapPodPath + device.Name() + constants.Bpf.Xsk_map
271+
logging.Debugf("mapping %s to %s", fullPath, containerMapPath)
269272
cresp.Mounts = append(cresp.Mounts, &pluginapi.Mount{
270273
HostPath: fullPath,
271-
ContainerPath: constants.Bpf.BpfMapPodPath,
274+
ContainerPath: containerMapPath,
272275
ReadOnly: false,
273276
})
274277
}
275278
}
276279

277-
envs[constants.Devices.EnvVarList] = strings.Join(crqt.DevicesIDs, " ")
280+
envVar := constants.Devices.EnvVarList + strings.ToUpper(pm.Name)
281+
envs[envVar] = strings.Join(crqt.DevicesIDs, " ")
278282
envsPrint, err := tools.PrettyString(envs)
279283
if err != nil {
280284
logging.Errorf("Error printing container environment variables: %v", err)
@@ -283,7 +287,6 @@ func (pm *PoolManager) Allocate(ctx context.Context,
283287
}
284288
cresp.Envs = envs
285289
response.ContainerResponses = append(response.ContainerResponses, cresp)
286-
287290
}
288291

289292
if !pm.UdsServerDisable {

‎internal/deviceplugin/poolManager_test.go

+47-20
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package deviceplugin
1818
import (
1919
"context"
2020
"encoding/json"
21+
"strings"
2122
"testing"
2223

2324
"github.com/intel/afxdp-plugins-for-kubernetes/constants"
@@ -57,6 +58,8 @@ func TestAllocate(t *testing.T) {
5758
pm.ServerFactory = udsserver.NewFakeServerFactory()
5859
pm.BpfHandler = bpf.NewFakeHandler()
5960

61+
envVar := constants.Devices.EnvVarList + strings.ToUpper(pm.Name)
62+
6063
testCases := []struct {
6164
name string
6265
containerRequests []*pluginapi.ContainerAllocateRequest
@@ -69,10 +72,10 @@ func TestAllocate(t *testing.T) {
6972
},
7073
expContainerResponses: []*pluginapi.ContainerAllocateResponse{
7174
{
72-
Envs: map[string]string{constants.Devices.EnvVarList: "dev_1"},
75+
Envs: map[string]string{envVar: "dev_1"},
7376
Mounts: []*pluginapi.Mount{
7477
{
75-
ContainerPath: constants.Uds.PodPath,
78+
ContainerPath: constants.Uds.PodPath + "dev_1" + constants.Uds.SockName,
7679
HostPath: "/tmp/fake-socket.sock",
7780
ReadOnly: false,
7881
},
@@ -90,10 +93,20 @@ func TestAllocate(t *testing.T) {
9093
},
9194
expContainerResponses: []*pluginapi.ContainerAllocateResponse{
9295
{
93-
Envs: map[string]string{constants.Devices.EnvVarList: "dev_1 dev_2 dev_3"},
96+
Envs: map[string]string{envVar: "dev_1 dev_2 dev_3"},
9497
Mounts: []*pluginapi.Mount{
9598
{
96-
ContainerPath: constants.Uds.PodPath,
99+
ContainerPath: constants.Uds.PodPath + "dev_1" + constants.Uds.SockName,
100+
HostPath: "/tmp/fake-socket.sock",
101+
ReadOnly: false,
102+
},
103+
{
104+
ContainerPath: constants.Uds.PodPath + "dev_2" + constants.Uds.SockName,
105+
HostPath: "/tmp/fake-socket.sock",
106+
ReadOnly: false,
107+
},
108+
{
109+
ContainerPath: constants.Uds.PodPath + "dev_3" + constants.Uds.SockName,
97110
HostPath: "/tmp/fake-socket.sock",
98111
ReadOnly: false,
99112
},
@@ -112,10 +125,10 @@ func TestAllocate(t *testing.T) {
112125
},
113126
expContainerResponses: []*pluginapi.ContainerAllocateResponse{
114127
{
115-
Envs: map[string]string{constants.Devices.EnvVarList: "dev_1"},
128+
Envs: map[string]string{envVar: "dev_1"},
116129
Mounts: []*pluginapi.Mount{
117130
{
118-
ContainerPath: constants.Uds.PodPath,
131+
ContainerPath: constants.Uds.PodPath + "dev_1" + constants.Uds.SockName,
119132
HostPath: "/tmp/fake-socket.sock",
120133
ReadOnly: false,
121134
},
@@ -124,10 +137,10 @@ func TestAllocate(t *testing.T) {
124137
Annotations: map[string]string{},
125138
},
126139
{
127-
Envs: map[string]string{constants.Devices.EnvVarList: "dev_2"},
140+
Envs: map[string]string{envVar: "dev_2"},
128141
Mounts: []*pluginapi.Mount{
129142
{
130-
ContainerPath: constants.Uds.PodPath,
143+
ContainerPath: constants.Uds.PodPath + "dev_2" + constants.Uds.SockName,
131144
HostPath: "/tmp/fake-socket.sock",
132145
ReadOnly: false,
133146
},
@@ -146,10 +159,20 @@ func TestAllocate(t *testing.T) {
146159
},
147160
expContainerResponses: []*pluginapi.ContainerAllocateResponse{
148161
{
149-
Envs: map[string]string{constants.Devices.EnvVarList: "dev_1 dev_2 dev_3"},
162+
Envs: map[string]string{envVar: "dev_1 dev_2 dev_3"},
150163
Mounts: []*pluginapi.Mount{
151164
{
152-
ContainerPath: constants.Uds.PodPath,
165+
ContainerPath: constants.Uds.PodPath + "dev_1" + constants.Uds.SockName,
166+
HostPath: "/tmp/fake-socket.sock",
167+
ReadOnly: false,
168+
},
169+
{
170+
ContainerPath: constants.Uds.PodPath + "dev_2" + constants.Uds.SockName,
171+
HostPath: "/tmp/fake-socket.sock",
172+
ReadOnly: false,
173+
},
174+
{
175+
ContainerPath: constants.Uds.PodPath + "dev_3" + constants.Uds.SockName,
153176
HostPath: "/tmp/fake-socket.sock",
154177
ReadOnly: false,
155178
},
@@ -158,10 +181,20 @@ func TestAllocate(t *testing.T) {
158181
Annotations: map[string]string{},
159182
},
160183
{
161-
Envs: map[string]string{constants.Devices.EnvVarList: "dev_4 dev_5 dev_6"},
184+
Envs: map[string]string{envVar: "dev_4 dev_5 dev_6"},
162185
Mounts: []*pluginapi.Mount{
163186
{
164-
ContainerPath: constants.Uds.PodPath,
187+
ContainerPath: constants.Uds.PodPath + "dev_4" + constants.Uds.SockName,
188+
HostPath: "/tmp/fake-socket.sock",
189+
ReadOnly: false,
190+
},
191+
{
192+
ContainerPath: constants.Uds.PodPath + "dev_5" + constants.Uds.SockName,
193+
HostPath: "/tmp/fake-socket.sock",
194+
ReadOnly: false,
195+
},
196+
{
197+
ContainerPath: constants.Uds.PodPath + "dev_6" + constants.Uds.SockName,
165198
HostPath: "/tmp/fake-socket.sock",
166199
ReadOnly: false,
167200
},
@@ -179,14 +212,8 @@ func TestAllocate(t *testing.T) {
179212
},
180213
expContainerResponses: []*pluginapi.ContainerAllocateResponse{
181214
{
182-
Envs: map[string]string{constants.Devices.EnvVarList: ""},
183-
Mounts: []*pluginapi.Mount{
184-
{
185-
ContainerPath: constants.Uds.PodPath,
186-
HostPath: "/tmp/fake-socket.sock",
187-
ReadOnly: false,
188-
},
189-
},
215+
Envs: map[string]string{envVar: ""},
216+
Mounts: []*pluginapi.Mount{},
190217
Devices: []*pluginapi.DeviceSpec{},
191218
Annotations: map[string]string{},
192219
},

0 commit comments

Comments
 (0)