Skip to content

Commit 065a057

Browse files
committed
dsa: dpdk example workload and use it in e2e
Signed-off-by: Tuomas Katila <[email protected]>
1 parent c5b7194 commit 065a057

File tree

8 files changed

+182
-26
lines changed

8 files changed

+182
-26
lines changed

.github/workflows/lib-e2e.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ jobs:
1818
- intel-dsa-plugin
1919
- intel-idxd-config-initcontainer
2020
- accel-config-demo
21+
- dsa-dpdk-dmadevtest
2122
- intel-deviceplugin-operator
2223
- name: e2e-iaa-gnr
2324
targetjob: e2e-iaa
@@ -44,6 +45,7 @@ jobs:
4445
- intel-dsa-plugin
4546
- intel-idxd-config-initcontainer
4647
- accel-config-demo
48+
- dsa-dpdk-dmadevtest
4749
- intel-deviceplugin-operator
4850
- intel-iaa-plugin
4951
- crypto-perf

README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -235,6 +235,7 @@ The summary of resources available via plugins in this repository is given in th
235235
* [dlb-libdlb-demo-pod.yaml](demo/dlb-libdlb-demo-pod.yaml)
236236
* `dsa.intel.com` : `wq-user-[shared or dedicated]`
237237
* [dsa-accel-config-demo-pod.yaml](demo/dsa-accel-config-demo-pod.yaml)
238+
* [dsa-dpdk-dmadevtest.yaml](demo/dsa-dpdk-dmadevtest.yaml)
238239
* `fpga.intel.com` : custom, see [mappings](cmd/fpga_admissionwebhook/README.md#mappings)
239240
* [intelfpga-job.yaml](demo/intelfpga-job.yaml)
240241
* `gpu.intel.com` : `i915`, `i915_monitoring`, `xe` or `xe_monitoring`

cmd/dsa_plugin/README.md

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ Table of Contents
77
* [Pre-built Images](#pre-built-images)
88
* [Verify Plugin Registration](#verify-plugin-registration)
99
* [Testing and Demos](#testing-and-demos)
10+
* [DPDK application](#dpdk-application)
1011

1112
## Introduction
1213

@@ -133,3 +134,24 @@ We can test the plugin is working by deploying the provided example accel-config
133134
---- ------ ---- ---- -------
134135
Warning FailedScheduling 2m26s default-scheduler 0/1 nodes are available: 1 Insufficient dsa.intel.com/wq-user-dedicated, 1 Insufficient dsa.intel.com/wq-user-shared.
135136
```
137+
### DPDK application
138+
139+
DPDK use requires a couple of changes for the DSA configuration. The workqueue mode has to be `dedicated` and the name has to have a prefix of `dpdk_`.
140+
141+
e.g.
142+
```
143+
{
144+
"dev":"wqX.0",
145+
"mode":"dedicated",
146+
"size":16,
147+
"group_id":0,
148+
"priority":10,
149+
"block_on_fault":1,
150+
"type":"user",
151+
"name":"dpdk_X0",
152+
"driver_name":"user",
153+
"threshold":15
154+
}
155+
```
156+
157+
For testing DPDK, one can use the test container in the [demo directory](../../demo/dsa-dpdk-dmadevtest/) and its deployment [file](../../demo/dsa-dpdk-dmadevtest.yaml).

demo/dsa-dpdk-dmadevtest.yaml

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
apiVersion: v1
2+
kind: Pod
3+
metadata:
4+
name: dpdk
5+
spec:
6+
restartPolicy: Never
7+
containers:
8+
- name: dpdk
9+
image: intel/dsa-dpdk-dmadevtest:devel
10+
securityContext:
11+
capabilities:
12+
add: ["SYS_RAWIO"]
13+
volumeMounts:
14+
- mountPath: /mnt/hugepage
15+
name: hugepage
16+
resources:
17+
requests:
18+
cpu: 4
19+
hugepages-2Mi: 1Gi
20+
memory: 1Gi
21+
dsa.intel.com/wq-user-dedicated: 2
22+
limits:
23+
cpu: 4
24+
hugepages-2Mi: 1Gi
25+
memory: 1Gi
26+
dsa.intel.com/wq-user-dedicated: 2
27+
volumes:
28+
- name: hugepage
29+
emptyDir:
30+
medium: HugePages

demo/dsa-dpdk-dmadevtest/Dockerfile

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
FROM debian:sid-slim AS builder
2+
3+
ARG DIR=/dpdk-build
4+
WORKDIR $DIR
5+
6+
RUN apt-get update && apt-get install -y --no-install-recommends wget build-essential meson ninja-build python3-pyelftools libnuma-dev python3-pip pkg-config dpkg-dev libipsec-mb-dev
7+
8+
# Download & unpack DPDK tarball
9+
ARG DPDK_TARBALL=dpdk-24.11.1.tar.xz
10+
ARG DPDK_TARBALL_SHA256="bcae7d42c449fc456dfb279feabcbe0599a29bebb2fe2905761e187339d96b8e"
11+
12+
RUN wget -q https://fast.dpdk.org/rel/$DPDK_TARBALL \
13+
&& echo "$DPDK_TARBALL_SHA256 $DPDK_TARBALL" | sha256sum -c - \
14+
&& tar -xf $DPDK_TARBALL && rm $DPDK_TARBALL
15+
16+
RUN cd dpdk-* && meson setup \
17+
"-Dplatform=generic" \
18+
"-Denable_drivers=dma/idxd" \
19+
"-Denable_apps=test" \
20+
--prefix $(pwd)/installdir \
21+
builddir
22+
RUN cd dpdk-* && ninja -C builddir install && install -D installdir/bin/dpdk-test /install_root/usr/bin/dpdk-test
23+
24+
FROM debian:sid-slim
25+
RUN apt-get update && apt-get install -y --no-install-recommends libipsec-mb2 libnuma1 libatomic1 && ldconfig -v
26+
COPY --from=builder /install_root/usr/bin/dpdk-test /usr/bin/dpdk-test
27+
28+
ENTRYPOINT ["dpdk-test", "dmadev_autotest"]

go.mod

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,6 @@ require (
130130
golang.org/x/time v0.7.0 // indirect
131131
golang.org/x/tools v0.28.0 // indirect
132132
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
133-
google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect
134133
google.golang.org/genproto/googleapis/api v0.0.0-20241202173237-19429a94021a // indirect
135134
google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a // indirect
136135
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -344,8 +344,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
344344
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
345345
gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
346346
gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
347-
google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY=
348-
google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo=
347+
google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ=
348+
google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80/go.mod h1:cc8bqMqtv9gMOr0zHg2Vzff5ULhhL2IXP4sbcn32Dro=
349349
google.golang.org/genproto/googleapis/api v0.0.0-20241202173237-19429a94021a h1:OAiGFfOiA0v9MRYsSidp3ubZaBnteRUyn3xB2ZQ5G/E=
350350
google.golang.org/genproto/googleapis/api v0.0.0-20241202173237-19429a94021a/go.mod h1:jehYqy3+AhJU9ve55aNOaSml7wUXjF9x6z2LcCfpAhY=
351351
google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a h1:hgh8P4EuoxpsuKMXX/To36nOFD7vixReXgn8lPGnt+o=

test/e2e/dsa/dsa.go

Lines changed: 97 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,10 @@ package dsa
1616

1717
import (
1818
"context"
19+
"io"
20+
"os"
1921
"path/filepath"
22+
"strings"
2023
"time"
2124

2225
"github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/utils"
@@ -36,13 +39,76 @@ const (
3639
kustomizationYaml = "deployments/dsa_plugin/overlays/dsa_initcontainer/dsa_initcontainer.yaml"
3740
configmapYaml = "demo/dsa.conf"
3841
demoYaml = "demo/dsa-accel-config-demo-pod.yaml"
42+
dpdkDemoYaml = "demo/dsa-dpdk-dmadevtest.yaml"
3943
podName = "dsa-accel-config-demo"
44+
dpdkPodName = "dpdk"
45+
46+
appMode = false
47+
dpdkMode = true
4048
)
4149

4250
func init() {
4351
ginkgo.Describe("DSA plugin [Device:dsa]", describe)
4452
}
4553

54+
func createDsaConfigWithDpdkMode(configmap, tmpDir string) string {
55+
origFile, err := os.Open(configmap)
56+
if err != nil {
57+
framework.Failf("unable to open dsa configmap from %s: %+v", configmap, err)
58+
}
59+
defer origFile.Close()
60+
61+
content, err := io.ReadAll(origFile)
62+
if err != nil {
63+
framework.Failf("unable to read dsa configmap from %s: %+v", configmap, err)
64+
}
65+
66+
updated := strings.ReplaceAll(string(content), "appX", "dpdk_X")
67+
68+
newconfigmap := filepath.Join(tmpDir, "dsa.conf")
69+
err = os.WriteFile(newconfigmap, []byte(updated), 0600)
70+
if err != nil {
71+
framework.Failf("unable to write updated dsa configmap: %+v", err)
72+
}
73+
74+
return newconfigmap
75+
}
76+
77+
func preparePlugin(dpdkMode bool, configmap, kustomizationPath string, ctx context.Context, f *framework.Framework) string {
78+
ginkgo.By("deploying DSA plugin")
79+
80+
if dpdkMode {
81+
tmpDir, err := os.MkdirTemp("", "dsa-config")
82+
if err != nil {
83+
framework.Failf("unable to create temporary dir for dsa config")
84+
}
85+
defer os.RemoveAll(tmpDir)
86+
87+
configmap = createDsaConfigWithDpdkMode(configmap, tmpDir)
88+
}
89+
90+
e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "create", "configmap", "intel-dsa-config", "--from-file="+configmap)
91+
e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "apply", "-k", filepath.Dir(kustomizationPath))
92+
93+
ginkgo.By("waiting for DSA plugin's availability")
94+
podList, err := e2epod.WaitForPodsWithLabelRunningReady(ctx, f.ClientSet, f.Namespace.Name,
95+
labels.Set{"app": "intel-dsa-plugin"}.AsSelector(), 1 /* one replica */, 300*time.Second)
96+
if err != nil {
97+
e2edebug.DumpAllNamespaceInfo(ctx, f.ClientSet, f.Namespace.Name)
98+
e2ekubectl.LogFailedContainers(ctx, f.ClientSet, f.Namespace.Name, framework.Logf)
99+
framework.Failf("unable to wait for all pods to be running and ready: %v", err)
100+
}
101+
102+
dpPodName := podList.Items[0].Name
103+
104+
ginkgo.By("checking DSA plugin's securityContext")
105+
if err = utils.TestPodsFileSystemInfo(podList.Items); err != nil {
106+
framework.Failf("container filesystem info checks failed: %v", err)
107+
}
108+
109+
return dpPodName
110+
}
111+
46112
func describe() {
47113
f := framework.NewDefaultFramework("dsaplugin")
48114
f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
@@ -62,29 +128,12 @@ func describe() {
62128
framework.Failf("unable to locate %q: %v", demoYaml, errFailedToLocateRepoFile)
63129
}
64130

65-
var dpPodName string
66-
67-
ginkgo.BeforeEach(func(ctx context.Context) {
68-
ginkgo.By("deploying DSA plugin")
69-
e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "create", "configmap", "intel-dsa-config", "--from-file="+configmap)
70-
71-
e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "apply", "-k", filepath.Dir(kustomizationPath))
72-
73-
ginkgo.By("waiting for DSA plugin's availability")
74-
podList, err := e2epod.WaitForPodsWithLabelRunningReady(ctx, f.ClientSet, f.Namespace.Name,
75-
labels.Set{"app": "intel-dsa-plugin"}.AsSelector(), 1 /* one replica */, 300*time.Second)
76-
if err != nil {
77-
e2edebug.DumpAllNamespaceInfo(ctx, f.ClientSet, f.Namespace.Name)
78-
e2ekubectl.LogFailedContainers(ctx, f.ClientSet, f.Namespace.Name, framework.Logf)
79-
framework.Failf("unable to wait for all pods to be running and ready: %v", err)
80-
}
81-
dpPodName = podList.Items[0].Name
131+
demoDpdkPath, errFailedToLocateRepoFile := utils.LocateRepoFile(dpdkDemoYaml)
132+
if errFailedToLocateRepoFile != nil {
133+
framework.Failf("unable to locate %q: %v", dpdkDemoYaml, errFailedToLocateRepoFile)
134+
}
82135

83-
ginkgo.By("checking DSA plugin's securityContext")
84-
if err = utils.TestPodsFileSystemInfo(podList.Items); err != nil {
85-
framework.Failf("container filesystem info checks failed: %v", err)
86-
}
87-
})
136+
var dpPodName string
88137

89138
ginkgo.AfterEach(func(ctx context.Context) {
90139
ginkgo.By("undeploying DSA plugin")
@@ -94,8 +143,10 @@ func describe() {
94143
}
95144
})
96145

97-
ginkgo.Context("When DSA resources are available [Resource:dedicated]", func() {
146+
ginkgo.Context("When DSA resources are available [Resource:dedicated] [Mode:app]", func() {
98147
ginkgo.BeforeEach(func(ctx context.Context) {
148+
dpPodName = preparePlugin(appMode, configmap, kustomizationPath, ctx, f)
149+
99150
ginkgo.By("checking if the resource is allocatable")
100151
if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, "dsa.intel.com/wq-user-dedicated", 300*time.Second, utils.WaitForPositiveResource); err != nil {
101152
framework.Failf("unable to wait for nodes to have positive allocatable resource: %v", err)
@@ -114,4 +165,27 @@ func describe() {
114165
ginkgo.It("does nothing", func() {})
115166
})
116167
})
168+
169+
ginkgo.Context("When DSA resources are available [Resource:dedicated] [Mode:dpdk]", func() {
170+
ginkgo.BeforeEach(func(ctx context.Context) {
171+
dpPodName = preparePlugin(dpdkMode, configmap, kustomizationPath, ctx, f)
172+
173+
ginkgo.By("checking if the resource is allocatable")
174+
if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, "dsa.intel.com/wq-user-dedicated", 300*time.Second, utils.WaitForPositiveResource); err != nil {
175+
framework.Failf("unable to wait for nodes to have positive allocatable resource: %v", err)
176+
}
177+
})
178+
179+
ginkgo.It("deploys a demo app [App:dpdk-test]", func(ctx context.Context) {
180+
e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "apply", "-f", demoDpdkPath)
181+
182+
ginkgo.By("waiting for the DSA DPDK demo to succeed")
183+
err := e2epod.WaitForPodSuccessInNamespaceTimeout(ctx, f.ClientSet, dpdkPodName, f.Namespace.Name, 200*time.Second)
184+
gomega.Expect(err).To(gomega.BeNil(), utils.GetPodLogs(ctx, f, dpdkPodName, dpdkPodName))
185+
})
186+
187+
ginkgo.When("there is no app to run [App:noapp]", func() {
188+
ginkgo.It("does nothing", func() {})
189+
})
190+
})
117191
}

0 commit comments

Comments
 (0)