From 1df91e2c8c936a887f34b5bd2d3afcb682b7c72c Mon Sep 17 00:00:00 2001 From: Zygmunt Krynicki Date: Tue, 26 Nov 2024 15:40:19 +0100 Subject: [PATCH] Third iteration of spread support - Tests defined in utils/test are now described by a task.yaml in the same directory and can run concurrently across many machines. - Tests for utils/ are now executed on openSUSE Tumbleweed since ttk themes is no longer a hard dependency in master. - Tests no longer run on openSUSE Leap 15.6 due to the age of default Python (3.6) and gcc/g++. The tight integration with SWIG which does not seem to support other Python versions very well. Perl hard-codes old GCC for extension modules. The upcoming openSUSE Leap 16 should be a viable target. In the meantime we can still test everything through rolling-release Tumbleweed. - Formatting of YAML files is now more uniform, at four spaces per tab. - The run-spread.sh script is now in the root of the tree. The script allows running all spread tests sequentially on one system, while collecting logs and artifacts for convenient analysis after the fact. - All systems are adjusted to run _four_ workers in parallel with _two_ virtual cores each and equipped with 1.5GB of virtual memory. This aims to best utilize the capacity of a typical CI worker with two to four cores and about 8GB of available memory. - Failing tests are marked as such, so that as a whole the entire spread suite can pass and be useful at catching regressions. Signed-off-by: Zygmunt Krynicki --- .gitignore | 3 + .image-garden.mk | 16 +- run-spread.sh | 29 +++ spread.yaml | 228 ++++++++++-------- tests/regression/apparmor/task.yaml | 204 ++++++++++------ tests/unit/binutils/task.yaml | 2 +- tests/unit/libapparmor/task.yaml | 2 +- tests/unit/parser-tst-caching/task.yaml | 2 +- tests/unit/parser-tst-dirtest/task.yaml | 2 +- tests/unit/parser-tst-equality/task.yaml | 6 +- tests/unit/parser-tst-error-output/task.yaml | 2 +- tests/unit/parser-tst-minimize/task.yaml | 2 +- tests/unit/parser-tst-parser-sanity/task.yaml | 6 +- tests/unit/parser/task.yaml | 22 +- tests/unit/utils/task.yaml | 10 - utils/test/task.yaml | 50 ++++ utils/vim/task.yaml | 3 + 17 files changed, 362 insertions(+), 227 deletions(-) create mode 100755 run-spread.sh delete mode 100644 tests/unit/utils/task.yaml create mode 100644 utils/test/task.yaml create mode 100644 utils/vim/task.yaml diff --git a/.gitignore b/.gitignore index 9fe00e383..13f1d2f89 100644 --- a/.gitignore +++ b/.gitignore @@ -315,3 +315,6 @@ tests/regression/apparmor/coredump *.qcow2 *.run .spread-reuse.yaml +.spread-reuse.*.yaml +spread-artifacts/ +spread-logs/ diff --git a/.image-garden.mk b/.image-garden.mk index eda9f2cc6..a93b65a94 100644 --- a/.image-garden.mk +++ b/.image-garden.mk @@ -30,18 +30,13 @@ packages: - python3-tk - python3-ttkthemes - swig -runcmd: -- apt clean endef # Ubuntu shares cloud-init profile with Debian. UBUNTU_CLOUD_INIT_USER_DATA_TEMPLATE=$(DEBIAN_CLOUD_INIT_USER_DATA_TEMPLATE) -# On openSUSE Leap the default gcc and python are very old. We can use more -# recent version of Python quite easily but perl extension module system does -# not want us to modify the CC that's baked into perl and all my attempts at -# using gcc-14 have failed. -define OPENSUSE_CLOUD_INIT_USER_DATA_TEMPLATE +# This is the cloud-init user-data profile for openSUSE Tumbleweed. +define OPENSUSE_tumbleweed_CLOUD_INIT_USER_DATA_TEMPLATE $(CLOUD_INIT_USER_DATA_TEMPLATE) packages: - attr @@ -67,12 +62,7 @@ packages: - python3-setuptools - python3-tk - python311 +- python3-devel - python311-devel - swig endef - -# On openSUSE tumbleweed the set of packages may drift towards more recent -# versions more rapidly than on Leap but the moment we want to, for example, -# move to Python 3.13, we can define a separate entry with different package -# set or perhaps with $(patsubst)-computed package set. -OPENSUSE_tumbleweed_CLOUD_INIT_USER_DATA_TEMPLATE=$(OPENSUSE_CLOUD_INIT_USER_DATA_TEMPLATE) diff --git a/run-spread.sh b/run-spread.sh new file mode 100755 index 000000000..da9ac89cf --- /dev/null +++ b/run-spread.sh @@ -0,0 +1,29 @@ +#!/bin/sh +# Run integration tests with spread sequentially on all the systems, using +# multiple workers per system. This mode is suitable to run on a single +# quad-core CPU with 8GB of RAM and no desktop session. +set -xeu + +if test -z "$(command -v spread)"; then + echo "You need to install spread from https://github.com/snapcore/spread with the Go compiler and the command: go install github.com/snapcore/spread/cmd/spread@latest" >&2 + exit 1 +fi + +if test -z "$(command -v image-garden)"; then + echo "You need to install image-garden from https://gitlab.com/zygoon/image-garden: make install prefix=/usr/local" >&2 + exit 1 +fi + +rm -rf spread-logs spread-artifacts +mkdir -p spread-logs +for system in \ + opensuse-cloud-tumbleweed \ + debian-cloud-12 \ + debian-cloud-13 \ + ubuntu-cloud-22.04 \ + ubuntu-cloud-24.04 \ + ubuntu-cloud-24.10; do + if ! spread -artifacts ./spread-artifacts -v "$system" | tee spread-logs/"$system".log; then + echo "Spread exited with code $?" >spread-logs/"$system".failed + fi +done diff --git a/spread.yaml b/spread.yaml index 71106cb89..d00cffba4 100644 --- a/spread.yaml +++ b/spread.yaml @@ -1,122 +1,138 @@ project: apparmor backends: - garden: - # The garden backend relies on https://gitlab.com/zygoon/image-garden - # TODO: Switch to a released version for better stability. - type: adhoc - # Use 2GB of RAM and four cores as otherwise we may not have enough memory - # to link the parser. It is better to have more workers than to have one - # big worker with lots of resources. - allocate: ADDRESS "$(QEMU_MEM_OPTION="-m 2048" QEMU_SMP_OPTION="-smp 4" image-garden allocate "$SPREAD_SYSTEM".$(uname -m))" - discard: image-garden discard "$SPREAD_SYSTEM_ADDRESS" - systems: - # All systems except for the one Ubuntu system are marked as manual. - # This way we don't accidentally spin up everything when someone runs - # spread without knowing better. - - opensuse-cloud-15.6: - username: opensuse - password: opensuse - workers: 2 - manual: true # Run only when explicitly named. - environment: - # openSUSE 15 ships very old default python. - PYTHON: /usr/bin/python3.11 - PYTHON_CONFIG: /usr/bin/python3.11-config - - opensuse-cloud-tumbleweed: - username: opensuse - password: opensuse - workers: 4 - manual: true - - debian-cloud-12: - username: debian - password: debian - workers: 4 - manual: true - - debian-cloud-13: - username: debian - password: debian - workers: 4 - manual: true - - ubuntu-cloud-22.04: - username: ubuntu - password: ubuntu - workers: 4 - manual: true - - ubuntu-cloud-24.04: - username: ubuntu - password: ubuntu - workers: 4 - manual: true - - ubuntu-cloud-24.10: - username: ubuntu - password: ubuntu - workers: 4 + garden: + # The garden backend relies on https://gitlab.com/zygoon/image-garden + # TODO: Switch to a released version for better stability. + type: adhoc + # Use just enough RAM to link the parser on a virtual system with two + # cores. Using more cores may easily consume more memory, due to make + # -j$(nproc, used below than a small CI/CD system is typically granted. + # It is better to have more workers than to have one big worker with + # lots of resources. + allocate: ADDRESS "$(QEMU_MEM_OPTION="-m 1536" QEMU_SMP_OPTION="-smp 2" image-garden allocate "$SPREAD_SYSTEM".$(uname -m))" + discard: image-garden discard "$SPREAD_SYSTEM_ADDRESS" + systems: + # All systems except for the one Ubuntu system are marked as + # manual. This way we don't accidentally spin up everything when + # someone runs spread without knowing better. + - opensuse-cloud-tumbleweed: + username: opensuse + password: opensuse + workers: 4 + manual: true + - debian-cloud-12: + username: debian + password: debian + workers: 4 + manual: true + - debian-cloud-13: + username: debian + password: debian + workers: 4 + manual: true + - ubuntu-cloud-22.04: + username: ubuntu + password: ubuntu + workers: 4 + manual: true + - ubuntu-cloud-24.04: + username: ubuntu + password: ubuntu + workers: 4 + manual: true + - ubuntu-cloud-24.10: + username: ubuntu + password: ubuntu + workers: 4 exclude: - - .git - - "*.o" - # Files related to spread and image-garden. - - "*.qcow2" - - "*.iso" - - "*.img" - - "*.log" - - "*.run" - - "*.lock" + - .git + - "*.o" + # Files related to spread and image-garden. + - "*.qcow2" + - "*.iso" + - "*.img" + - "*.log" + - "*.run" + - "*.lock" + - spread-logs + - spread-artifacts # Copy the project to this path on the test system. # This is also available as $SPREAD_PATH. path: /tmp/apparmor prepare: | - # Configure libapparmor but only if a makefile is not already present. - # This makes repeated iteration with -reuse much faster, as the chain of - # invocations of make below are efficient if nothing needs to be done. - if [ ! -f "$SPREAD_PATH"/libraries/libapparmor/Makefile ]; then - ( - cd $SPREAD_PATH/libraries/libapparmor - sh ./autogen.sh && sh ./configure --prefix=/usr --with-perl --with-python - ) - fi - # Build libapparmor. - make -C $SPREAD_PATH/libraries/libapparmor -j"$(nproc)" - # Build apparmor_parser. - make -C $SPREAD_PATH/parser -j"$(nproc)" - # Build binary utilities (aa-exec and firends). - make -C $SPREAD_PATH/binutils -j"$(nproc)" - # Build python utilities. - make -C $SPREAD_PATH/utils -j"$(nproc)" + # Configure libapparmor but only if a makefile is not already present. + # This makes repeated iteration with -reuse much faster, as the chain of + # invocations of make below are efficient if nothing needs to be done. + if [ ! -f "$SPREAD_PATH"/libraries/libapparmor/Makefile ]; then + ( + cd $SPREAD_PATH/libraries/libapparmor + sh ./autogen.sh && sh ./configure --prefix=/usr --with-perl --with-python + ) + fi + # Build libapparmor. + make -C $SPREAD_PATH/libraries/libapparmor -j"$(nproc)" + # Build apparmor_parser. + make -C $SPREAD_PATH/parser -j"$(nproc)" + # Build binary utilities (aa-exec and firends). + make -C $SPREAD_PATH/binutils -j"$(nproc)" + # Build python utilities. + make -C $SPREAD_PATH/utils -j"$(nproc)" # In case of failure, include the kernel version in the log. debug-each: | - uname -a + uname -a suites: - tests/unit/: - summary: Unit tests that do not exercise the kernel layer. - tests/regression/: - summary: Regression tests for parser-kernel interaction. - prepare: | - # Spread does not support programmatically generated test variants. - # Ensure that the list baked into tests/regressin/apparmor/task.yaml - # contains all the tests defined in tests/regression/apparmor/Makefile. - echo '$(foreach t,$(TESTS),$(info TEST/$t))' | \ - make -n -f "$SPREAD_PATH"/tests/regression/apparmor/Makefile -f /dev/stdin | \ - grep -F TEST/ | \ - cut -d / -f 2 | \ - tee apparmor-regression-tests.txt - fail=0 - for V in $(cat apparmor-regression-tests.txt); do - if ! grep -xF ' TEST/'"$V"': 1' "$SPREAD_PATH"/tests/regression/apparmor/task.yaml; then - echo "tests/regression/task.yaml: missing test variant: TEST/$V" >&2 - fail=1 - fi - done - if [ "$fail" -ne 0 ]; then - echo "exiting due to missing variants listed above" >&2 - exit 1 - fi + utils/: + summary: Unit tests for the Python utilities. + prepare: | + # Generate apparmor profiles that the tests rely on. + make -C "$SPREAD_PATH"/parser/tst gen_xtrans gen_dbus + # Spread does not support programmatically generated test variants. + # Ensure that the list baked into utils/test/task.yaml contains all + # the files matching utils/test/test-*.py + fail=0 + for V in $SPREAD_PATH/utils/test/test-*.py; do + Vdash="$(basename "$V" | sed -e 's,^test-,,' -e 's,\.py$,,')" + Vunder="$(basename "$V" | sed -e 's,^test-,,' -e 's,\.py$,,' -e 's,-,_,g')" + if ! grep -xF ' TEST/'"$Vunder"': '"$Vdash" "$SPREAD_PATH"/utils/test/task.yaml; then + echo "utils/test/task.yaml: missing test variant: TEST/$Vunder: $Vdash" >&2 + fail=1 + fi + done + if [ "$fail" -ne 0 ]; then + echo "exiting due to missing variants listed above" >&2 + exit 1 + fi + tests/unit/: + summary: Unit tests that do not exercise the kernel layer. + tests/regression/: + summary: Regression tests for parser-kernel interaction. + prepare: | + # Spread does not support programmatically generated test variants. + # Ensure that the list baked into tests/regression/apparmor/task.yaml + # contains all the tests defined in tests/regression/apparmor/Makefile. + echo '$(foreach t,$(TESTS),$(info TEST/$t))' | \ + make -n -f "$SPREAD_PATH"/tests/regression/apparmor/Makefile -f /dev/stdin | \ + grep -F TEST/ | \ + cut -d / -f 2 | \ + tee apparmor-regression-tests.txt + fail=0 + while read -r V; do + if ! grep -xF ' TEST/'"$V"': 1' "$SPREAD_PATH"/tests/regression/apparmor/task.yaml; then + echo "tests/regression/task.yaml: missing test variant: TEST/$V" >&2 + fail=1 + fi + done &2 + exit 1 + fi - # Build all the apparmor regression test programs. - make -C "$SPREAD_PATH"/tests/regression/apparmor -j"$(nproc)" - restore: | - rm -f apparmor-regression-tests.txt + # Build all the apparmor regression test programs. + make -C "$SPREAD_PATH"/tests/regression/apparmor -j"$(nproc)" + restore: | + rm -f apparmor-regression-tests.txt diff --git a/tests/regression/apparmor/task.yaml b/tests/regression/apparmor/task.yaml index 4e6d795c8..e73cd8a83 100644 --- a/tests/regression/apparmor/task.yaml +++ b/tests/regression/apparmor/task.yaml @@ -1,83 +1,129 @@ summary: run all of the apparmor regression test suite # See spread.yaml for the code which verifies that nothing is missing from this list of variants environment: - TEST/aa_exec: 1 - TEST/access: 1 - TEST/attach_disconnected: 1 - TEST/at_secure: 1 - TEST/introspect: 1 - TEST/capabilities: 1 - TEST/changeprofile: 1 - TEST/onexec: 1 - TEST/changehat: 1 - TEST/changehat_fork: 1 - TEST/changehat_misc: 1 - TEST/chdir: 1 - TEST/clone: 1 - TEST/complain: 1 - TEST/coredump: 1 - TEST/deleted: 1 - TEST/e2e: 1 - TEST/environ: 1 - TEST/exec: 1 - TEST/exec_qual: 1 - TEST/fchdir: 1 - TEST/fd_inheritance: 1 - TEST/fork: 1 - TEST/i18n: 1 - TEST/link: 1 - TEST/link_subset: 1 - TEST/mkdir: 1 - TEST/mmap: 1 - TEST/mount: 1 - TEST/mult_mount: 1 - TEST/named_pipe: 1 - TEST/namespaces: 1 - TEST/net_raw: 1 - TEST/open: 1 - TEST/openat: 1 - TEST/pipe: 1 - TEST/pivot_root: 1 - TEST/posix_ipc: 1 - TEST/ptrace: 1 - TEST/pwrite: 1 - TEST/query_label: 1 - TEST/regex: 1 - TEST/rename: 1 - TEST/readdir: 1 - TEST/rw: 1 - TEST/socketpair: 1 - TEST/swap: 1 - TEST/sd_flags: 1 - TEST/setattr: 1 - TEST/symlink: 1 - TEST/syscall: 1 - TEST/sysv_ipc: 1 - TEST/tcp: 1 - TEST/unix_fd_server: 1 - TEST/unix_socket_pathname: 1 - TEST/unix_socket_abstract: 1 - TEST/unix_socket_unnamed: 1 - TEST/unix_socket_autobind: 1 - TEST/unlink: 1 - TEST/userns: 1 - TEST/xattrs: 1 - TEST/longpath: 1 - TEST/nfs: 1 - TEST/xattrs_profile: 1 - TEST/dbus_eavesdrop: 1 - TEST/dbus_message: 1 - TEST/dbus_service: 1 - TEST/dbus_unrequested_reply: 1 - TEST/io_uring: 1 - TEST/exec_stack: 1 - TEST/aa_policy_cache: 1 - TEST/nnp: 1 - TEST/stackonexec: 1 - TEST/stackprofile: 1 + TEST/aa_exec: 1 + TEST/aa_policy_cache: 1 + TEST/access: 1 + TEST/at_secure: 1 + TEST/attach_disconnected: 1 + TEST/capabilities: 1 + TEST/changehat: 1 + TEST/changehat_fork: 1 + TEST/changehat_misc: 1 + TEST/changeprofile: 1 + TEST/chdir: 1 + TEST/clone: 1 + TEST/complain: 1 + TEST/coredump: 1 + TEST/dbus_eavesdrop: 1 + TEST/dbus_message: 1 + TEST/dbus_service: 1 + TEST/dbus_unrequested_reply: 1 + TEST/deleted: 1 + TEST/e2e: 1 + TEST/environ: 1 + TEST/exec: 1 + TEST/exec_qual: 1 + TEST/exec_stack: 1 + TEST/fchdir: 1 + TEST/fd_inheritance: 1 + TEST/fork: 1 + TEST/i18n: 1 + TEST/introspect: 1 + TEST/io_uring: 1 + TEST/link: 1 + TEST/link_subset: 1 + TEST/longpath: 1 + TEST/mkdir: 1 + TEST/mmap: 1 + TEST/mount: 1 + TEST/mult_mount: 1 + TEST/named_pipe: 1 + TEST/namespaces: 1 + TEST/net_raw: 1 + TEST/nfs: 1 + TEST/nnp: 1 + TEST/onexec: 1 + TEST/open: 1 + TEST/openat: 1 + TEST/pipe: 1 + TEST/pivot_root: 1 + TEST/posix_ipc: 1 + TEST/ptrace: 1 + TEST/pwrite: 1 + TEST/query_label: 1 + TEST/readdir: 1 + TEST/regex: 1 + TEST/rename: 1 + TEST/rw: 1 + TEST/sd_flags: 1 + TEST/setattr: 1 + TEST/socketpair: 1 + TEST/stackonexec: 1 + TEST/stackprofile: 1 + TEST/swap: 1 + TEST/symlink: 1 + TEST/syscall: 1 + TEST/sysv_ipc: 1 + TEST/tcp: 1 + TEST/unix_fd_server: 1 + TEST/unix_socket_abstract: 1 + TEST/unix_socket_autobind: 1 + TEST/unix_socket_pathname: 1 + TEST/unix_socket_unnamed: 1 + TEST/unlink: 1 + TEST/userns: 1 + TEST/xattrs: 1 + TEST/xattrs_profile: 1 + # Some tests are currently failing. Those are listed below. For each variant + # listed above, the XFAIL variable contains a list of spread systems where this + # test is expected to fail. + # + # Error: unix_fd_server passed. Test 'ATTACH_DISCONNECTED (attach_disconnected.path rule at /)' was expected to 'fail' + XFAIL/attach_disconnected: opensuse-cloud-tumbleweed debian-cloud-12 debian-cloud-13 ubuntu-cloud-22.04 + # Error: unix_fd_server failed. Test 'fd passing; unconfined client' was expected to 'pass'. Reason for failure 'FAIL - bind failed: Permission denied' + # Error: unix_fd_server failed. Test 'fd passing; confined client w/ rw' was expected to 'pass'. Reason for failure 'FAIL - bind failed: Permission denied' + XFAIL/deleted: opensuse-cloud-tumbleweed debian-cloud-12 debian-cloud-13 + # Error: unix_fd_server failed. Test 'fd passing; confined -> unconfined' was expected to 'pass'. Reason for failure 'FAIL - bind failed: Permission denied' + # Error: unix_fd_server failed. Test 'fd passing; unconfined -> confined' was expected to 'pass'. Reason for failure 'FAIL CLIENT - connect Permission denied' + # Error: unix_fd_server failed. Test 'fd passing; unconfined -> confined (no perm)' was expected to 'pass'. Reason for failure 'FAIL CLIENT - connect Permission denied' + # Error: unix_fd_server failed. Test 'fd passing; confined -> confined' was expected to 'pass'. Reason for failure 'FAIL - bind failed: Permission denied' + XFAIL/unix_fd_server: opensuse-cloud-tumbleweed debian-cloud-12 debian-cloud-13 + # Error: unix_socket failed. Test 'AF_UNIX pathname socket (stream); confined server w/ access (rw)' was expected to 'pass'. Reason for failure 'FAIL - setsockopt (SO_RCVTIMEO): Permission denied' + # Error: unix_socket passed. Test 'AF_UNIX pathname socket (stream); confined server w/ a missing af_unix access (create)' was expected to 'fail' + # Error: unix_socket failed. Test 'AF_UNIX pathname socket (stream); confined client w/ access (rw)' was expected to 'pass'. Reason for failure 'FAIL - setsockopt (SO_RCVTIMEO): Permission denied' + # xpass: AF_UNIX pathname socket (dgram); confined server w/ access (rw) + # Error: unix_socket passed. Test 'AF_UNIX pathname socket (dgram); confined server w/ a missing af_unix access (create)' was expected to 'fail' + # xpass: AF_UNIX pathname socket (dgram); confined client w/ access (rw) + # Error: unix_socket failed. Test 'AF_UNIX pathname socket (seqpacket); confined server w/ access (rw)' was expected to 'pass'. Reason for failure 'FAIL - setsockopt (SO_RCVTIMEO): Permission denied' + # Error: unix_socket passed. Test 'AF_UNIX pathname socket (seqpacket); confined server w/ a missing af_unix access (create)' was expected to 'fail' + # Error: unix_socket failed. Test 'AF_UNIX pathname socket (seqpacket); confined client w/ access (rw)' was expected to 'pass'. Reason for failure 'FAIL - setsockopt (SO_RCVTIMEO): Permission denied' + XFAIL/unix_socket_pathname: opensuse-cloud-tumbleweed debian-cloud-12 debian-cloud-13 +artifacts: + - bash.log + - bash.err execute: | - # Run the shell script that is named after the spread variant we are running - # now. The makefile runs them all sequentially via the "alltests" goal. Here - # we can parallelize it through spread and also have a way to run precisely - # the test we want, especially for debugging. - bash "$SPREAD_VARIANT".sh + # Run the shell script that is named after the spread variant we are running + # now. The makefile runs them all sequentially via the "alltests" goal. Here + # we can parallelize it through spread and also have a way to run precisely + # the test we want, especially for debugging. + if ! bash -x "$SPREAD_VARIANT".sh >bash.log 2>bash.err; then + for xfail in ${XFAIL:-}; do + if [ "$SPREAD_SYSTEM" = "$xfail" ]; then + echo "Ignoring expected failure of $SPREAD_TASK on $SPREAD_SYSTEM" + exit 0 + fi + done + + echo "Test $SPREAD_VARIANT has unexpectedly failed" + echo "Test execution logs are in the files bash.{log.err} and are collected as artifacts" + exit 1 + else + for xfail in ${XFAIL:-}; do + if [ "$SPREAD_SYSTEM" = "$xfail" ]; then + echo "Test $SPREAD_VARIANT has unexpectedly passed" + echo "Test execution logs are in the files bash.{log.err} and are collected as artifacts" + exit 1 + fi + done + fi diff --git a/tests/unit/binutils/task.yaml b/tests/unit/binutils/task.yaml index 33bda2559..a9b0973a3 100644 --- a/tests/unit/binutils/task.yaml +++ b/tests/unit/binutils/task.yaml @@ -1,3 +1,3 @@ summary: Run unit tests of AppArmor binary utilities execute: | - make -C "$SPREAD_PATH/binutils" check + make -C "$SPREAD_PATH"/binutils check diff --git a/tests/unit/libapparmor/task.yaml b/tests/unit/libapparmor/task.yaml index dc0dbaad6..cd45b46ff 100644 --- a/tests/unit/libapparmor/task.yaml +++ b/tests/unit/libapparmor/task.yaml @@ -1,3 +1,3 @@ summary: Run unit tests of libapparmor execute: | - make -C "$SPREAD_PATH/libraries/libapparmor" check + make -C "$SPREAD_PATH"/libraries/libapparmor check diff --git a/tests/unit/parser-tst-caching/task.yaml b/tests/unit/parser-tst-caching/task.yaml index ed729ad9e..8ffa062fa 100644 --- a/tests/unit/parser-tst-caching/task.yaml +++ b/tests/unit/parser-tst-caching/task.yaml @@ -1,3 +1,3 @@ summary: Run apparmor_parser caching test from parser/tst execute: | - make -C "$SPREAD_PATH/parser/tst" -j"$(nproc)" caching + make -C "$SPREAD_PATH"/parser/tst -j"$(nproc)" caching diff --git a/tests/unit/parser-tst-dirtest/task.yaml b/tests/unit/parser-tst-dirtest/task.yaml index eb399a263..f68449c88 100644 --- a/tests/unit/parser-tst-dirtest/task.yaml +++ b/tests/unit/parser-tst-dirtest/task.yaml @@ -1,3 +1,3 @@ summary: Run apparmor_parser dirtest test from parser/tst execute: | - make -C "$SPREAD_PATH/parser/tst" -j"$(nproc)" dirtest + make -C "$SPREAD_PATH"/parser/tst -j"$(nproc)" dirtest diff --git a/tests/unit/parser-tst-equality/task.yaml b/tests/unit/parser-tst-equality/task.yaml index 5401e97ec..cb45b3562 100644 --- a/tests/unit/parser-tst-equality/task.yaml +++ b/tests/unit/parser-tst-equality/task.yaml @@ -1,7 +1,11 @@ summary: Run apparmor_parser tests from parser/tst # This test is particularly slow. Those values are aimed at running fast enough # on a Raspberry Pi 5, representing a slower-ish computer. +# +# TODO: figure out how to make this test more parallelizable. warn-timeout: 20m kill-timeout: 30m +# Start this task sooner rather than later. +priority: 100 execute: | - make -C "$SPREAD_PATH/parser/tst" -j"$(nproc)" equality + make -C "$SPREAD_PATH"/parser/tst -j"$(nproc)" equality diff --git a/tests/unit/parser-tst-error-output/task.yaml b/tests/unit/parser-tst-error-output/task.yaml index f22f91413..8c2ae0e13 100644 --- a/tests/unit/parser-tst-error-output/task.yaml +++ b/tests/unit/parser-tst-error-output/task.yaml @@ -1,3 +1,3 @@ summary: Run apparmor_parser error_output test from parser/tst execute: | - make -C "$SPREAD_PATH/parser/tst" -j"$(nproc)" error_output + make -C "$SPREAD_PATH"/parser/tst -j"$(nproc)" error_output diff --git a/tests/unit/parser-tst-minimize/task.yaml b/tests/unit/parser-tst-minimize/task.yaml index 6f1e43951..28e9fb401 100644 --- a/tests/unit/parser-tst-minimize/task.yaml +++ b/tests/unit/parser-tst-minimize/task.yaml @@ -1,3 +1,3 @@ summary: Run apparmor_parser minimize test from parser/tst execute: | - make -C "$SPREAD_PATH/parser/tst" -j"$(nproc)" minimize + make -C "$SPREAD_PATH"/parser/tst -j"$(nproc)" minimize diff --git a/tests/unit/parser-tst-parser-sanity/task.yaml b/tests/unit/parser-tst-parser-sanity/task.yaml index 6c5fa34e8..5b6e8196f 100644 --- a/tests/unit/parser-tst-parser-sanity/task.yaml +++ b/tests/unit/parser-tst-parser-sanity/task.yaml @@ -1,7 +1,11 @@ summary: Run apparmor_parser parser_sanity test from parser/tst # This test is particularly slow. Those values are aimed at running fast enough # on a Raspberry Pi 5, representing a slower-ish computer. +# +# TODO: figure out how to make this test more parallelizable. warn-timeout: 20m kill-timeout: 30m +# Start this task sooner rather than later. +priority: 100 execute: | - make -C "$SPREAD_PATH/parser/tst" -j"$(nproc)" parser_sanity + make -C "$SPREAD_PATH"/parser/tst -j"$(nproc)" parser_sanity diff --git a/tests/unit/parser/task.yaml b/tests/unit/parser/task.yaml index 034368346..0eb1a7170 100644 --- a/tests/unit/parser/task.yaml +++ b/tests/unit/parser/task.yaml @@ -1,16 +1,16 @@ summary: Run apparmor_parser unit tests from parser/ details: | - The parser has a number of different tests. Those are all represented as - spread task variants so that they are directly visisble and runnable. + The parser has a number of different tests. Those are all represented as + spread task variants so that they are directly visible and runnable. environment: - TEST/tst_regex: 1 - TEST/tst_misc: 1 - TEST/tst_symtab: 1 - TEST/tst_variable: 1 - TEST/tst_lib: 1 + TEST/tst_regex: 1 + TEST/tst_misc: 1 + TEST/tst_symtab: 1 + TEST/tst_variable: 1 + TEST/tst_lib: 1 prepare: | - # The test relies on make to build a binary. - make -C "$SPREAD_PATH/parser" -j"$(nproc)" "$SPREAD_VARIANT" + # The test relies on make to build a binary. + make -C "$SPREAD_PATH"/parser -j"$(nproc)" "$SPREAD_VARIANT" execute: | - cd "$SPREAD_PATH"/parser - ./"$SPREAD_VARIANT" + cd "$SPREAD_PATH"/parser + ./"$SPREAD_VARIANT" diff --git a/tests/unit/utils/task.yaml b/tests/unit/utils/task.yaml deleted file mode 100644 index cac37f28f..000000000 --- a/tests/unit/utils/task.yaml +++ /dev/null @@ -1,10 +0,0 @@ -summary: Run unit tests of python utilities -# FIXME: Exclude openSUSE as the test depends on python3-ttkthemes -# which are not packaged in the distribution. -systems: - - -opensuse-* -prepare: | - # Generate the profiles that the test relies on. - make -C "$SPREAD_PATH"/parser/tst gen_xtrans gen_dbus -execute: | - make -C "$SPREAD_PATH"/utils check diff --git a/utils/test/task.yaml b/utils/test/task.yaml new file mode 100644 index 000000000..1595f8e6b --- /dev/null +++ b/utils/test/task.yaml @@ -0,0 +1,50 @@ +summary: Run "make check" broken down by test name +systems: + - -opensuse-cloud-15.6 +environment: + # FIXME: Spread disallows dashes in variant names, most likely as a bug. + # As such we map the all_under_score variant name to actual test name. + TEST/aa_cli_bootstrap: aa-cli-bootstrap + TEST/aa_decode: aa-decode + TEST/aa_easyprof: aa-easyprof + TEST/aa_notify: aa-notify + TEST/aa: aa + TEST/aare: aare + TEST/abi: abi + TEST/alias: alias + TEST/all: all + TEST/baserule: baserule + TEST/boolean: boolean + TEST/capability: capability + TEST/change_profile: change_profile + TEST/common: common + TEST/config: config + TEST/dbus: dbus + TEST/example: example + TEST/file: file + TEST/include: include + TEST/io_uring: io_uring + TEST/libapparmor_test_multi: libapparmor-test_multi + TEST/logparser: logparser + TEST/logprof: logprof + TEST/minitools: minitools + TEST/mount: mount + TEST/mqueue: mqueue + TEST/network: network + TEST/notify: notify + TEST/parser_simple_tests: parser-simple-tests + TEST/pivot_root: pivot_root + TEST/profile_list: profile-list + TEST/profile_storage: profile-storage + TEST/profiles: profiles + TEST/ptrace: ptrace + TEST/regex_matches: regex_matches + TEST/rlimit: rlimit + TEST/severity: severity + TEST/signal: signal + TEST/translations: translations + TEST/unix: unix + TEST/userns: userns + TEST/variable: variable +execute: | + make --warn-undefined-variables check-one-test-"$TEST" diff --git a/utils/vim/task.yaml b/utils/vim/task.yaml new file mode 100644 index 000000000..dfe75c2d7 --- /dev/null +++ b/utils/vim/task.yaml @@ -0,0 +1,3 @@ +summary: Run "make check" +execute: | + make --warn-undefined-variables check