Skip to content

Commit 4909ea8

Browse files
author
Kernel Patches Daemon
committed
adding ci files
1 parent 2d311f4 commit 4909ea8

23 files changed

+1449
-18
lines changed

‎.github/workflows/lint.yml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
name: "lint"
2+
3+
on:
4+
pull_request:
5+
push:
6+
branches:
7+
- master
8+
9+
jobs:
10+
shellcheck:
11+
# This workflow gets injected into other Linux repositories, but we don't
12+
# want it to run there.
13+
if: ${{ github.repository == 'kernel-patches/vmtest' }}
14+
name: ShellCheck
15+
runs-on: ubuntu-latest
16+
steps:
17+
- name: Checkout repository
18+
uses: actions/checkout@v3
19+
- name: Run ShellCheck
20+
uses: ludeeus/action-shellcheck@master
21+
env:
22+
SHELLCHECK_OPTS: --severity=warning --exclude=SC1091

‎.github/workflows/test.yml

Lines changed: 349 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,349 @@
1+
name: bpf-ci
2+
3+
on:
4+
pull_request:
5+
push:
6+
branches:
7+
- bpf_base
8+
- bpf-next_base
9+
10+
concurrency:
11+
group: ci-test-${{ github.ref_name }}
12+
cancel-in-progress: true
13+
14+
jobs:
15+
llvm-toolchain:
16+
runs-on: ubuntu-latest
17+
outputs:
18+
llvm: ${{ steps.llvm-toolchain-impl.outputs.version }}
19+
steps:
20+
- id: llvm-version
21+
uses: libbpf/ci/get-llvm-version@master
22+
- id: llvm-toolchain-impl
23+
shell: bash
24+
run: echo "version=llvm-${{ steps.llvm-version.outputs.version }}" >> $GITHUB_OUTPUT
25+
set-matrix:
26+
needs: llvm-toolchain
27+
runs-on: ubuntu-latest
28+
outputs:
29+
build-matrix: ${{ steps.set-matrix-impl.outputs.build_matrix }}
30+
test-matrix: ${{ steps.set-matrix-impl.outputs.test_matrix }}
31+
steps:
32+
- id: set-matrix-impl
33+
shell: python3 -I {0}
34+
run: |
35+
from json import dumps
36+
from enum import Enum
37+
import os
38+
39+
class Arch(Enum):
40+
"""
41+
CPU architecture supported by CI.
42+
"""
43+
aarch64 = "aarch64"
44+
s390x = "s390x"
45+
x86_64 = "x86_64"
46+
47+
def set_output(name, value):
48+
"""Write an output variable to the GitHub output file."""
49+
with open(os.getenv("GITHUB_OUTPUT"), "a") as f:
50+
f.write(f"{name}={value}\n")
51+
52+
def generate_test_config(test):
53+
"""Create the configuration for the provided test."""
54+
experimental = test.endswith("_parallel")
55+
config = {
56+
"test": test,
57+
"continue_on_error": experimental,
58+
# While in experimental mode, parallel jobs may get stuck
59+
# anywhere, including in user space where the kernel won't detect
60+
# a problem and panic. We add a second layer of (smaller) timeouts
61+
# here such that if we get stuck in a parallel run, we hit this
62+
# timeout and fail without affecting the overall job success (as
63+
# would be the case if we hit the job-wide timeout). For
64+
# non-experimental jobs, 360 is the default which will be
65+
# superseded by the overall workflow timeout (but we need to
66+
# specify something).
67+
"timeout_minutes": 30 if experimental else 360,
68+
}
69+
return config
70+
71+
matrix = [
72+
{"kernel": "LATEST", "runs_on": [], "arch": Arch.x86_64.value, "toolchain": "gcc"},
73+
{"kernel": "LATEST", "runs_on": [], "arch": Arch.x86_64.value, "toolchain": "${{ needs.llvm-toolchain.outputs.llvm }}"},
74+
{"kernel": "LATEST", "runs_on": [], "arch": Arch.aarch64.value, "toolchain": "gcc"},
75+
{"kernel": "LATEST", "runs_on": [], "arch": Arch.aarch64.value, "toolchain": "${{ needs.llvm-toolchain.outputs.llvm }}"},
76+
{"kernel": "LATEST", "runs_on": [], "arch": Arch.s390x.value, "toolchain": "gcc"},
77+
]
78+
self_hosted_repos = [
79+
"kernel-patches/bpf",
80+
"kernel-patches/vmtest",
81+
]
82+
83+
# Only a few repository within "kernel-patches" use self-hosted runners.
84+
if "${{ github.repository_owner }}" != "kernel-patches" or "${{ github.repository }}" not in self_hosted_repos:
85+
# Outside of those repositories, we only run on x86_64 GH hosted runners (ubuntu-latest)
86+
for idx in range(len(matrix) - 1, -1, -1):
87+
if matrix[idx]["arch"] != Arch.x86_64.value:
88+
del matrix[idx]
89+
else:
90+
matrix[idx]["runs_on"] = ["ubuntu-latest"]
91+
else:
92+
# Otherwise, run on (self-hosted, arch) runners
93+
for idx in range(len(matrix) - 1, -1, -1):
94+
matrix[idx]["runs_on"].extend(["self-hosted", matrix[idx]["arch"]])
95+
96+
build_matrix = {"include": matrix}
97+
set_output("build_matrix", dumps(build_matrix))
98+
99+
tests = [
100+
"test_progs",
101+
"test_progs_parallel",
102+
"test_progs_no_alu32",
103+
"test_progs_no_alu32_parallel",
104+
"test_maps",
105+
"test_verifier",
106+
]
107+
test_matrix = {"include": [{**config, **generate_test_config(test)}
108+
for config in matrix
109+
for test in tests]}
110+
set_output("test_matrix", dumps(test_matrix))
111+
build:
112+
name: build for ${{ matrix.arch }} with ${{ matrix.toolchain }}
113+
needs: set-matrix
114+
runs-on: ${{ matrix.runs_on }}
115+
timeout-minutes: 100
116+
strategy:
117+
fail-fast: false
118+
matrix: ${{ fromJSON(needs.set-matrix.outputs.build-matrix) }}
119+
env:
120+
KERNEL: ${{ matrix.kernel }}
121+
REPO_ROOT: ${{ github.workspace }}
122+
REPO_PATH: ""
123+
KBUILD_OUTPUT: kbuild-output/
124+
steps:
125+
- uses: actions/checkout@v3
126+
# We fetch an actual bit of history here to facilitate incremental
127+
# builds (which may check out some earlier upstream change).
128+
with:
129+
fetch-depth: 50
130+
- if: ${{ github.repository == 'kernel-patches/vmtest' }}
131+
name: Download bpf-next tree
132+
uses: libbpf/ci/get-linux-source@master
133+
with:
134+
dest: '.kernel'
135+
- if: ${{ github.repository == 'kernel-patches/vmtest' }}
136+
name: Move linux source in place
137+
shell: bash
138+
run: |
139+
rm -rf .kernel/.git
140+
cp -rf .kernel/. .
141+
rm -rf .kernel
142+
- name: Get commit meta-data
143+
id: get-commit-metadata
144+
shell: bash
145+
run: |
146+
if [ ${{ github.event_name }} = 'push' ]; then
147+
branch="${{ github.ref_name }}"
148+
echo "branch=${branch}" >> "${GITHUB_OUTPUT}"
149+
else
150+
branch="${{ github.base_ref }}"
151+
echo "branch=${branch}" >> "${GITHUB_OUTPUT}"
152+
fi
153+
154+
upstream=$(echo "${branch}" | sed 's@_base$@@')
155+
commit="$(
156+
git rev-parse "origin/${upstream}" &> /dev/null \
157+
|| (
158+
git fetch --quiet --prune --no-tags --depth=1 --no-recurse-submodules origin +refs/heads/${upstream}:refs/remotes/origin/${upstream} \
159+
&& git rev-parse "origin/${upstream}"
160+
)
161+
)"
162+
163+
echo "timestamp=$(TZ=utc git show --format='%cd' --no-patch --date=iso-strict-local ${commit})" >> "${GITHUB_OUTPUT}"
164+
echo "commit=${commit}" >> "${GITHUB_OUTPUT}"
165+
echo "Most recent upstream commit is ${commit}"
166+
- name: Pull recent KBUILD_OUTPUT contents
167+
uses: actions/cache@v3
168+
with:
169+
path: ${{ env.KBUILD_OUTPUT }}
170+
key: kbuild-output-${{ matrix.arch }}-${{ matrix.toolchain }}-${{ steps.get-commit-metadata.outputs.branch }}-${{ steps.get-commit-metadata.outputs.timestamp }}-${{ steps.get-commit-metadata.outputs.commit }}
171+
restore-keys: |
172+
kbuild-output-${{ matrix.arch }}-${{ matrix.toolchain }}-${{ steps.get-commit-metadata.outputs.branch }}-${{ steps.get-commit-metadata.outputs.timestamp }}-
173+
kbuild-output-${{ matrix.arch }}-${{ matrix.toolchain }}-${{ steps.get-commit-metadata.outputs.branch }}-
174+
kbuild-output-${{ matrix.arch }}-${{ matrix.toolchain }}-
175+
- name: Prepare incremental build
176+
shell: bash
177+
run: |
178+
set -e -u
179+
180+
# $1 - the SHA-1 to fetch and check out
181+
fetch_and_checkout() {
182+
local build_base_sha="${1}"
183+
184+
# If cached artifacts became stale for one reason or another, we
185+
# may not have the build base SHA available. Fetch it and retry.
186+
git fetch origin "${build_base_sha}" && git checkout --quiet "${build_base_sha}"
187+
}
188+
189+
# $1 - value of KBUILD_OUTPUT
190+
clear_cache_artifacts() {
191+
local kbuild_output="${1}"
192+
echo "Unable to find earlier upstream ref. Discarding KBUILD_OUTPUT contents..."
193+
rm --recursive --force "${kbuild_output}"
194+
mkdir "${kbuild_output}"
195+
false
196+
}
197+
198+
# $1 - value of KBUILD_OUTPUT
199+
# $2 - current time in ISO 8601 format
200+
restore_source_code_times() {
201+
local kbuild_output="${1}"
202+
local current_time="${2}"
203+
local src_time="$(date --iso-8601=ns --date="${current_time} - 2 minutes")"
204+
local obj_time="$(date --iso-8601=ns --date="${current_time} - 1 minute")"
205+
206+
git ls-files | xargs --max-args=10000 touch -m --no-create --date="${src_time}"
207+
find "${kbuild_output}" -type f | xargs --max-args=10000 touch -m --no-create --date="${obj_time}"
208+
git checkout --quiet -
209+
echo "Adjusted src and obj time stamps relative to system time"
210+
}
211+
212+
mkdir --parents "${KBUILD_OUTPUT}"
213+
current_time="$(date --iso-8601=ns)"
214+
215+
if [ -f "${KBUILD_OUTPUT}/.build-base-sha" ]; then
216+
build_base_sha="$(cat "${KBUILD_OUTPUT}/.build-base-sha")"
217+
echo "Setting up base build state for ${build_base_sha}"
218+
219+
(
220+
git checkout --quiet "${build_base_sha}" \
221+
|| fetch_and_checkout "${build_base_sha}" \
222+
|| clear_cache_artifacts "${KBUILD_OUTPUT}"
223+
) && restore_source_code_times "${KBUILD_OUTPUT}" "${current_time}"
224+
else
225+
echo "No previous build data found"
226+
fi
227+
228+
echo -n "${{ steps.get-commit-metadata.outputs.commit }}" > "${KBUILD_OUTPUT}/.build-base-sha"
229+
- uses: libbpf/ci/patch-kernel@master
230+
with:
231+
patches-root: '${{ github.workspace }}/ci/diffs'
232+
repo-root: '${{ github.workspace }}'
233+
- name: Setup build environment
234+
uses: libbpf/ci/setup-build-env@master
235+
- name: Build kernel image
236+
uses: libbpf/ci/build-linux@master
237+
with:
238+
arch: ${{ matrix.arch }}
239+
toolchain: ${{ matrix.toolchain }}
240+
kbuild-output: ${{ env.KBUILD_OUTPUT }}
241+
max-make-jobs: 32
242+
- if: ${{ github.event_name != 'push' }}
243+
name: Build selftests
244+
uses: libbpf/ci/build-selftests@master
245+
with:
246+
toolchain: ${{ matrix.toolchain }}
247+
kbuild-output: ${{ env.KBUILD_OUTPUT }}
248+
max-make-jobs: 32
249+
- if: ${{ github.event_name != 'push' }}
250+
name: Build samples
251+
uses: libbpf/ci/build-samples@master
252+
with:
253+
toolchain: ${{ matrix.toolchain }}
254+
kbuild-output: ${{ env.KBUILD_OUTPUT }}
255+
max-make-jobs: 32
256+
- if: ${{ github.event_name != 'push' }}
257+
name: Tar artifacts
258+
run: |
259+
# Remove intermediate object files that we have no use for. Ideally
260+
# we'd just exclude them from tar below, but it does not provide
261+
# options to express the precise constraints.
262+
find selftests/ -name "*.o" -a ! -name "*.bpf.o" -print0 | \
263+
xargs --null --max-args=10000 rm
264+
265+
# Strip debug information, which is excessively large (consuming
266+
# bandwidth) while not actually being used (the kernel does not use
267+
# DWARF to symbolize stacktraces).
268+
strip --strip-debug "${KBUILD_OUTPUT}"/vmlinux
269+
270+
file_list=""
271+
if [ "${{ github.repository }}" == "kernel-patches/vmtest" ]; then
272+
# Package up a bunch of additional infrastructure to support running
273+
# 'make kernelrelease' and bpf tool checks later on.
274+
file_list="$(find . -iname Makefile | xargs) \
275+
scripts/ \
276+
tools/testing/selftests/bpf/ \
277+
tools/include/ \
278+
tools/bpf/bpftool/";
279+
fi
280+
# zstd is installed by default in the runner images.
281+
tar -cf - \
282+
"${KBUILD_OUTPUT}"/.config \
283+
"${KBUILD_OUTPUT}"/$(KBUILD_OUTPUT="${KBUILD_OUTPUT}" make -s image_name) \
284+
"${KBUILD_OUTPUT}"/include/config/auto.conf \
285+
"${KBUILD_OUTPUT}"/include/generated/autoconf.h \
286+
"${KBUILD_OUTPUT}"/vmlinux \
287+
${file_list} \
288+
--exclude '*.cmd' \
289+
--exclude '*.d' \
290+
--exclude '*.h' \
291+
--exclude '*.output' \
292+
selftests/bpf/ | zstd -T0 -19 -o vmlinux-${{ matrix.arch }}-${{ matrix.toolchain }}.tar.zst
293+
- if: ${{ github.event_name != 'push' }}
294+
name: Remove KBUILD_OUTPUT contents
295+
shell: bash
296+
run: |
297+
# Remove $KBUILD_OUTPUT to prevent cache creation for pull requests.
298+
# Only on pushed changes are build artifacts actually cached, because
299+
# of github.com/actions/cache's cache isolation logic.
300+
rm -rf "${KBUILD_OUTPUT}"
301+
- if: ${{ github.event_name != 'push' }}
302+
uses: actions/upload-artifact@v3
303+
with:
304+
name: vmlinux-${{ matrix.arch }}-${{ matrix.toolchain }}
305+
if-no-files-found: error
306+
path: vmlinux-${{ matrix.arch }}-${{ matrix.toolchain }}.tar.zst
307+
test:
308+
if: ${{ github.event_name != 'push' }}
309+
name: ${{ matrix.test }} on ${{ matrix.arch }} with ${{ matrix.toolchain }}
310+
needs: [set-matrix, build]
311+
strategy:
312+
fail-fast: false
313+
matrix: ${{ fromJSON(needs.set-matrix.outputs.test-matrix) }}
314+
runs-on: ${{ matrix.runs_on }}
315+
timeout-minutes: 100
316+
env:
317+
KERNEL: ${{ matrix.kernel }}
318+
REPO_ROOT: ${{ github.workspace }}
319+
REPO_PATH: ""
320+
KBUILD_OUTPUT: kbuild-output/
321+
steps:
322+
- uses: actions/checkout@v3
323+
- uses: actions/download-artifact@v3
324+
with:
325+
name: vmlinux-${{ matrix.arch }}-${{ matrix.toolchain }}
326+
path: .
327+
- name: Untar artifacts
328+
# zstd is installed by default in the runner images.
329+
run: zstd -d -T0 vmlinux-${{ matrix.arch }}-${{ matrix.toolchain }}.tar.zst --stdout | tar -xf -
330+
- name: Prepare rootfs
331+
uses: libbpf/ci/prepare-rootfs@master
332+
with:
333+
project-name: 'libbpf'
334+
arch: ${{ matrix.arch }}
335+
kernel: ${{ matrix.kernel }}
336+
kernel-root: '.'
337+
kbuild-output: ${{ env.KBUILD_OUTPUT }}
338+
image-output: '/tmp/root.img'
339+
test: ${{ matrix.test }}
340+
- name: Run selftests
341+
uses: libbpf/ci/run-qemu@master
342+
continue-on-error: ${{ matrix.continue_on_error }}
343+
timeout-minutes: ${{ matrix.timeout_minutes }}
344+
with:
345+
arch: ${{ matrix.arch}}
346+
img: '/tmp/root.img'
347+
vmlinuz: '${{ github.workspace }}/vmlinuz'
348+
kernel-root: '.'
349+
max-cpu: 8

‎README

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +0,0 @@
1-
Linux kernel
2-
============
3-
4-
There are several guides for kernel developers and users. These guides can
5-
be rendered in a number of formats, like HTML and PDF. Please read
6-
Documentation/admin-guide/README.rst first.
7-
8-
In order to build the documentation, use ``make htmldocs`` or
9-
``make pdfdocs``. The formatted documentation can also be read online at:
10-
11-
https://www.kernel.org/doc/html/latest/
12-
13-
There are various text files in the Documentation/ subdirectory,
14-
several of them using the Restructured Text markup notation.
15-
16-
Please read the Documentation/process/changes.rst file, as it contains the
17-
requirements for building and running the kernel, and information about
18-
the problems which may result by upgrading your kernel.

‎ci/diffs/.keep

Whitespace-only changes.

0 commit comments

Comments
 (0)