Merge tests: run regression tests with spread (self-hosted)

This requires a runner with the tags: linux, x86_64, kvm. One needs to
be provisioned for the AppArmor project for the pipeline to function.

It is possible to run the same tests on SAAS runners offered by GitLab
but due to issue gitlab-org/gitlab-runner#6208 there is no way to expose
/dev/kvm on the host to the guest. Without this feature emulation works
but is rather slow as to be impractical.

Note that there's some overlap between the build-all job and spread that
might be avoided in the future. At present this is made more difficult
by the fact that the path where build-all job builds libapparmor is
stored internally by autotools. This prevents us from using GitLab
artifacts from moving the built files across to the spread testing jobs
without extra work.

In addition to adding the spread job, remove test-build-regression job.
This job is now redundant since the same operation is done when spread
builds and runs regression tests.

Signed-off-by: Zygmunt Krynicki <zygmunt.krynicki@canonical.com>

MR: https://gitlab.com/apparmor/apparmor/-/merge_requests/1512
Approved-by: John Johansen <john@jjmx.net>
Merged-by: John Johansen <john@jjmx.net>

.gitlab-ci.yml

@@ -13,6 +13,7 @@ workflow:
 stages:
   - build
   - test
+  - spread
 
 .ubuntu-common:
   before_script:
@@ -126,19 +127,6 @@ test-profiles:
     - make -C profiles check-abstractions.d
     - make -C profiles check-local
 
-# Build the regression tests (don't run them because that needs kernel access)
-test-build-regression:
-  stage: test
-  needs: ["build-all"]
-  extends:
-    - .ubuntu-common
-  script:
-    # Additional dependencies required by regression tests
-    - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" install_extra_deps "Installing additional dependencies..."
-    - apt-get install --no-install-recommends -y attr fuse-overlayfs libdbus-1-dev liburing-dev
-    - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" install_extra_deps
-    - make -C tests/regression/apparmor -j $(nproc)
-
 shellcheck:
   stage: test
   needs: []
@@ -196,3 +184,123 @@ coverity:
       - "apparmor-*.tar.gz"
   rules:
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_PATH == "apparmor/apparmor"
+
+.image-garden-x86_64:
+  stage: spread
+  # TODO: use tagged release once container tagging is improved upstream.
+  image: registry.gitlab.com/zygoon/image-garden:latest
+  tags:
+    - linux
+    - x86_64
+    - kvm
+  variables:
+    ARCH: x86_64
+    GARDEN_DL_DIR: dl
+    CACHE_POLICY: pull-push
+    CACHE_COMPRESSION_LEVEL: fastest
+  before_script:
+    # Prepare the image in dry-run mode. This helps in debugging cache misses
+    # when files are not cached correctly by the runner, causing the build section
+    # below to always do hevy-duty work.
+    - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" prepare_image_dry_run "Prepare image (dry run)"
+    - image-garden make --dry-run --debug "$GARDEN_SYSTEM.$ARCH.run" "$GARDEN_SYSTEM.$ARCH.qcow2" "$GARDEN_SYSTEM.seed.iso" "$GARDEN_SYSTEM.user-data" "$GARDEN_SYSTEM.meta-data"
+    - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" prepare_image_dry_run
+  script:
+    # Prepare the image, for real.
+    - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" prepare_image "Prepare image"
+    - image-garden make "$GARDEN_SYSTEM.$ARCH.run" "$GARDEN_SYSTEM.$ARCH.qcow2" "$GARDEN_SYSTEM.seed.iso" "$GARDEN_SYSTEM.user-data" "$GARDEN_SYSTEM.meta-data"
+    - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" prepare_image
+  cache:
+    # Cache the base image (pre-customization).
+    - key: image-garden-base-${GARDEN_SYSTEM}.${ARCH}
+      policy: $CACHE_POLICY
+      when: always
+      paths:
+        - $GARDEN_DL_DIR
+        # Those are never mutated so they are safe to share.
+        - efi-code.*.img
+        - efi-vars.*.img
+    # Cache the customized system. This cache depends on .image-garden.mk file
+    # so that any customization updates are immediately acted upon.
+    - key:
+        prefix: image-garden-custom-${GARDEN_SYSTEM}.${ARCH}-
+        files:
+          - .image-garden.mk
+      policy: $CACHE_POLICY
+      when: always
+      paths:
+        - $GARDEN_SYSTEM.*
+        - $GARDEN_SYSTEM.seed.iso
+        - $GARDEN_SYSTEM.meta-data
+        - $GARDEN_SYSTEM.user-data
+
+# This job builds and caches the image that the job below looks at.
+image-ubuntu-cloud-24.04-x86_64:
+  extends: .image-garden-x86_64
+  variables:
+    GARDEN_SYSTEM: ubuntu-cloud-24.04
+  needs: []
+  dependencies: []
+  rules:
+    - if: $CI_COMMIT_TAG
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH
+      changes:
+        paths:
+          - .image-garden.mk
+          - .gitlab-ci.yml
+        compare_to: "refs/heads/master"
+
+.spread-x86_64:
+  extends: .image-garden-x86_64
+  variables:
+    # GitLab project identifier of zygoon/spread-dist can be seen on
+    # https://gitlab.com/zygoon/spread-dist, under the three-dot menu on
+    # top-right.
+    SPREAD_GITLAB_PROJECT_ID: "65375371"
+    # Git revision of spread to install.
+    # This must have been built via spread-dist.
+    # TODO: switch to upstream 1.0 release when available.
+    SPREAD_REV: 413817eda7bec07a3885e0717c178b965f8924e1
+    # Run all the tasks for a given system.
+    SPREAD_ARGS: "garden:$GARDEN_SYSTEM:"
+    SPREAD_GOARCH: amd64
+  before_script:
+    # Prepare the image in dry-run mode. This helps in debugging cache misses
+    # when files are not cached correctly by the runner, causing the build section
+    # below to always do hevy-duty work.
+    - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" prepare_image_dry_run "Prepare image (dry run)"
+    - image-garden make --dry-run --debug "$GARDEN_SYSTEM.$ARCH.run" "$GARDEN_SYSTEM.$ARCH.qcow2" "$GARDEN_SYSTEM.seed.iso" "$GARDEN_SYSTEM.user-data" "$GARDEN_SYSTEM.meta-data"
+    - stat .image-garden.mk "$GARDEN_SYSTEM".* || true
+    - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" prepare_image_dry_run
+    # Install the selected revision of spread.
+    - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" install_spread "Installing spread..."
+    # Install pre-built spread from https://gitlab.com/zygoon/spread-dist generic package repository.
+    - |
+      curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --location --output spread "${CI_API_V4_URL}/projects/${SPREAD_GITLAB_PROJECT_ID}/packages/generic/spread/${SPREAD_REV}/spread.${SPREAD_GOARCH}"
+    - chmod +x spread
+    - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" install_spread
+  script:
+    - printf '\e[0K%s:%s:%s\r\e[0K%s\n' section_start "$(date +%s)" run_spread "Running spread for $GARDEN_SYSTEM..."
+    # TODO: transform to inject ^...$ to properly select jobs to run.
+    - mkdir -p spread-logs spread-artifacts
+    - ./spread -list $SPREAD_ARGS |
+      split --number=l/"${CI_NODE_INDEX:-1}"/"${CI_NODE_TOTAL:-1}" |
+      xargs --verbose ./spread -v -artifacts ./spread-artifacts -v | tee spread-logs/"$GARDEN_SYSTEM".log
+    - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" run_spread
+  artifacts:
+    paths:
+      - spread-logs
+      - spread-artifacts
+    when: always
+
+spread-ubuntu-cloud-24.04-x86_64:
+  extends: .spread-x86_64
+  variables:
+    GARDEN_SYSTEM: ubuntu-cloud-24.04
+    SPREAD_ARGS: garden:$GARDEN_SYSTEM:tests/regression/ garden:$GARDEN_SYSTEM:tests/profiles/
+    CACHE_POLICY: pull
+  dependencies: []
+  needs:
+    - job: image-ubuntu-cloud-24.04-x86_64
+      optional: true
+  parallel: 4

spread.yaml

@@ -66,7 +66,6 @@ backends:
             - ubuntu-cloud-24.04:
                   username: ubuntu
                   password: ubuntu
-                  workers: 4
                   manual: true
             - ubuntu-cloud-24.10:
                   username: ubuntu