Second iteration of spread support

Compared to v1 the following improvements have been made:

- The cost of installing packages have been shifted from each startup to image
  preparation phase, thanks to the integration of custom cloud-init profiles
  into image-garden. This has dramatic impact on iteration time while also
  entirely removing requirement to be online to run once a prepared image is
  available.

- Support for running on Google Compute Engine has been removed since it would
  not be able to use cloud-init the same way would currently only complicate
  setup.

- The number of workers have been tuned for local iteration, aiming for
  comfortable work with 16GB of memory on the host. Once CI/CD pipeline
  support is introduced I will add a dedicated entry so that resources are
  utilized well both locally and when running in CI.

- The set of regression tests listed in tests/regression/apparmor/task.yaml is
  now cross-checked so introduction of a new test to the makefile there is
  automatically flagged and causes spread to fail with a clear message.

- The task tests/unit/utils has been improved to generate profiles. Thanks to
  Christian Boltz for explaining this relationship between tests.

- A number of comments have been improved and cleaned up for readability,
  accuracy and sometimes better grammar.

Signed-off-by: Zygmunt Krynicki <zygmunt.krynicki@canonical.com>

.gitignore

@@ -306,9 +306,11 @@ tests/regression/apparmor/xattrs_profile
 tests/regression/apparmor/coredump
 **/__pycache__/
 *.orig
+
 # Patterns related to spread integration tests
 *.img
 *.iso
+*.lock
 *.log
 *.qcow2
 *.run

.image-garden.mk

@@ -0,0 +1,78 @@
+# This file is read by image-garden when spread is allocating test machines.
+# All the package installation happens through cloud-init profiles defined
+# below.
+
+# This is the cloud-init user-data profile for all Debian systems. Note that it
+# is an extension of the default profile necessary for operation of
+# image-garden.
+define DEBIAN_CLOUD_INIT_USER_DATA_TEMPLATE
+$(CLOUD_INIT_USER_DATA_TEMPLATE)
+packages:
+- attr
+- autoconf
+- autoconf-archive
+- automake
+- bison
+- build-essential
+- dejagnu
+- flake8
+- flex
+- gettext
+- libdbus-1-dev
+- libtool
+- liburing-dev
+- pkg-config
+- python3-all-dev
+- python3-gi
+- python3-notify2
+- python3-psutil
+- python3-setuptools
+- python3-tk
+- python3-ttkthemes
+- swig
+runcmd:
+- apt clean
+endef
+
+# Ubuntu shares cloud-init profile with Debian.
+UBUNTU_CLOUD_INIT_USER_DATA_TEMPLATE=$(DEBIAN_CLOUD_INIT_USER_DATA_TEMPLATE)
+
+# On openSUSE Leap the default gcc and python are very old. We can use more
+# recent version of Python quite easily but perl extension module system does
+# not want us to modify the CC that's baked into perl and all my attempts at
+# using gcc-14 have failed.
+define OPENSUSE_CLOUD_INIT_USER_DATA_TEMPLATE
+$(CLOUD_INIT_USER_DATA_TEMPLATE)
+packages:
+- attr
+- autoconf
+- autoconf-archive
+- automake
+- bison
+- dbus-1-devel
+- dejagnu
+- flex
+- gcc
+- gcc-c++
+- gettext
+- gobject-introspection
+- libtool
+- liburing2-devel
+- make
+- pkg-config
+- python3-flake8
+- python3-notify2
+- python3-psutil
+- python3-setuptools
+- python3-setuptools
+- python3-tk
+- python311
+- python311-devel
+- swig
+endef
+
+# On openSUSE tumbleweed the set of packages may drift towards more recent
+# versions more rapidly than on Leap but the moment we want to, for example,
+# move to Python 3.13, we can define a separate entry with different package
+# set or perhaps with $(patsubst)-computed package set.
+OPENSUSE_tumbleweed_CLOUD_INIT_USER_DATA_TEMPLATE=$(OPENSUSE_CLOUD_INIT_USER_DATA_TEMPLATE)

spread.yaml

@@ -1,21 +1,5 @@
 project: apparmor
 backends:
-  google:
-    key: '$(HOST: echo "$SPREAD_GOOGLE_KEY")'
-    halt-timeout: 1h
-    # Run only when explicitly named. This backend requires a Google Compute
-    # Engine (GCE) account and incurs cost on every use. It is most practical
-    # to scale-out tests once spread can express sufficient concurrency.
-    manual: true
-    # TODO: This needs to be adjusted to properly account for apparmor tests.
-    location: snapd-spread/europe-west2-b
-    systems:
-      - ubuntu-22.04-64:
-          workers: 4
-      - ubuntu-24.04-64:
-          workers: 4
-      - ubuntu-24.10-64:
-          workers: 4
   garden:
     # The garden backend relies on https://gitlab.com/zygoon/image-garden
     # TODO: Switch to a released version for better stability.
@@ -41,127 +25,61 @@ backends:
       - opensuse-cloud-tumbleweed:
           username: opensuse
           password: opensuse
-          workers: 2
+          workers: 4
           manual: true
       - debian-cloud-12:
           username: debian
           password: debian
-          workers: 2
+          workers: 4
           manual: true
       - debian-cloud-13:
           username: debian
           password: debian
-          workers: 2
+          workers: 4
           manual: true
       - ubuntu-cloud-22.04:
           username: ubuntu
           password: ubuntu
-          workers: 2
+          workers: 4
           manual: true
       - ubuntu-cloud-24.04:
           username: ubuntu
           password: ubuntu
-          workers: 2
+          workers: 4
           manual: true
       - ubuntu-cloud-24.10:
           username: ubuntu
           password: ubuntu
-          workers: 2
+          workers: 4
 
 exclude:
   - .git
+  - "*.o"
+  # Files related to spread and image-garden.
   - "*.qcow2"
   - "*.iso"
   - "*.img"
   - "*.log"
   - "*.run"
+  - "*.lock"
 
 # Copy the project to this path on the test system.
 # This is also available as $SPREAD_PATH.
 path: /tmp/apparmor
 
 prepare: |
-  # Install build dependencies, depending on the type of system running.
+  # Configure libapparmor but only if a makefile is not already present.
-  case "$SPREAD_SYSTEM" in
+  # This makes repeated iteration with -reuse much faster, as the chain of
-    debian-*|ubuntu-*)
+  # invocations of make below are efficient if nothing needs to be done.
-      apt-get update -qq
+  if [ ! -f "$SPREAD_PATH"/libraries/libapparmor/Makefile ]; then
-      # TODO: extract this from README.md libapparmor section and unifiy with what is in .gitlab-ci.yml.
+    (
-      DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+      cd $SPREAD_PATH/libraries/libapparmor
-        attr \
+      sh ./autogen.sh && sh ./configure --prefix=/usr --with-perl --with-python
-        autoconf \
+    )
-        autoconf-archive \
+  fi
-        automake \
-        bison \
-        build-essential \
-        dejagnu \
-        flake8 \
-        flex \
-        gettext \
-        libdbus-1-dev \
-        libtool \
-        liburing-dev \
-        pkg-config \
-        python3-all-dev \
-        python3-gi \
-        python3-notify2 \
-        python3-psutil \
-        python3-setuptools \
-        python3-tk \
-        python3-ttkthemes \
-        swig
-    ;;
-    opensuse-*)
-      # On openSUSE the default gcc and python are very old. We can use more
-      # recent version of Python quite easily but perl extension module system
-      # does not want us to modify the CC that's baked into perl and all my
-      # attempts at using gcc-14 have failed.
-      zypper install -y \
-        attr \
-        autoconf \
-        autoconf-archive \
-        automake \
-        bison \
-        dbus-1-devel \
-        dejagnu \
-        flex \
-        gcc \
-        gcc-c++ \
-        gettext \
-        gobject-introspection \
-        libtool \
-        liburing2-devel \
-        make \
-        pkg-config \
-        python3-flake8 \
-        python3-notify2 \
-        python3-psutil \
-        python3-setuptools \
-        python3-setuptools \
-        python3-tk \
-        python311 \
-        python311-devel \
-        swig
-    ;;
-    *)
-      echo "Please add support for $SPREAD_SYSTEM to spread.yaml"
-      exit 1
-    ;;
-  esac
-
-  # TODO: add logic to skip this build phase and use prebuild binaries from
-  # GitLab pipeline. This should also reduce the number of dependencies we need
-  # to install above.
-
-  # Configure libapparmor. We have to pass CC and CXX explicitly if provided in
-  # the environment.
-  (
-    cd $SPREAD_PATH/libraries/libapparmor
-    sh ./autogen.sh && sh ./configure --prefix=/usr --with-perl --with-python
-  )
   # Build libapparmor.
   make -C $SPREAD_PATH/libraries/libapparmor -j"$(nproc)"
   # Build apparmor_parser.
-  # The alternative builds sequentially to use less memory.
   make -C $SPREAD_PATH/parser -j"$(nproc)"
   # Build binary utilities (aa-exec and firends).
   make -C $SPREAD_PATH/binutils -j"$(nproc)"
@@ -178,5 +96,27 @@ suites:
   tests/regression/:
     summary: Regression tests for parser-kernel interaction.
     prepare: |
-      # FIXME: `make -C tests/regression` does not do anything.
+      # Spread does not support programmatically generated test variants.
-      make -C "$SPREAD_PATH/tests/regression/apparmor" -j"$(nproc)"
+      # Ensure that the list baked into tests/regressin/apparmor/task.yaml
+      # contains all the tests defined in tests/regression/apparmor/Makefile.
+      echo '$(foreach t,$(TESTS),$(info TEST/$t))' | \
+        make -n -f "$SPREAD_PATH"/tests/regression/apparmor/Makefile -f /dev/stdin | \
+        grep -F TEST/ | \
+        cut -d / -f 2 | \
+        tee apparmor-regression-tests.txt
+      fail=0
+      for V in $(cat apparmor-regression-tests.txt); do
+        if ! grep -xF '  TEST/'"$V"': 1' "$SPREAD_PATH"/tests/regression/apparmor/task.yaml; then
+          echo "tests/regression/task.yaml: missing test variant: TEST/$V" >&2
+          fail=1
+        fi
+      done
+      if [ "$fail" -ne 0 ]; then
+        echo "exiting due to missing variants listed above" >&2
+        exit 1
+      fi
+
+      # Build all the apparmor regression test programs.
+      make -C "$SPREAD_PATH"/tests/regression/apparmor -j"$(nproc)"
+    restore: |
+      rm -f apparmor-regression-tests.txt

tests/regression/apparmor/task.yaml

@@ -1,8 +1,5 @@
 summary: run all of the apparmor regression test suite
-# TODO: This is somewhat tedious and it'd be nicer if we could somehow generate
+# See spread.yaml for the code which verifies that nothing is missing from this list of variants
-# the variants without doing it "by hand" with the following one-liner:
-#
-# echo '$(foreach t,$(TESTS),$(info TEST/$t: 1))'| make -f Makefile -f /dev/stdin
 environment:
   TEST/aa_exec: 1
   TEST/access: 1

tests/unit/utils/task.yaml

@@ -3,5 +3,8 @@ summary: Run unit tests of python utilities
 # which are not packaged in the distribution.
 systems:
  - -opensuse-*
+prepare: |
+  # Generate the profiles that the test relies on.
+  make -C "$SPREAD_PATH"/parser/tst gen_xtrans gen_dbus
 execute: |
-  make -C "$SPREAD_PATH/utils" check
+  make -C "$SPREAD_PATH"/utils check