diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cb107af0b..393f6c72c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,6 +13,7 @@ workflow: stages: - build - test + - spread .ubuntu-common: before_script: @@ -126,19 +127,6 @@ test-profiles: - make -C profiles check-abstractions.d - make -C profiles check-local -# Build the regression tests (don't run them because that needs kernel access) -test-build-regression: - stage: test - needs: ["build-all"] - extends: - - .ubuntu-common - script: - # Additional dependencies required by regression tests - - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" install_extra_deps "Installing additional dependencies..." - - apt-get install --no-install-recommends -y attr fuse-overlayfs libdbus-1-dev liburing-dev - - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" install_extra_deps - - make -C tests/regression/apparmor -j $(nproc) - shellcheck: stage: test needs: [] @@ -196,3 +184,123 @@ coverity: - "apparmor-*.tar.gz" rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_PATH == "apparmor/apparmor" + +.image-garden-x86_64: + stage: spread + # TODO: use tagged release once container tagging is improved upstream. + image: registry.gitlab.com/zygoon/image-garden:latest + tags: + - linux + - x86_64 + - kvm + variables: + ARCH: x86_64 + GARDEN_DL_DIR: dl + CACHE_POLICY: pull-push + CACHE_COMPRESSION_LEVEL: fastest + before_script: + # Prepare the image in dry-run mode. This helps in debugging cache misses + # when files are not cached correctly by the runner, causing the build section + # below to always do hevy-duty work. + - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" prepare_image_dry_run "Prepare image (dry run)" + - image-garden make --dry-run --debug "$GARDEN_SYSTEM.$ARCH.run" "$GARDEN_SYSTEM.$ARCH.qcow2" "$GARDEN_SYSTEM.seed.iso" "$GARDEN_SYSTEM.user-data" "$GARDEN_SYSTEM.meta-data" + - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" prepare_image_dry_run + script: + # Prepare the image, for real. + - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" prepare_image "Prepare image" + - image-garden make "$GARDEN_SYSTEM.$ARCH.run" "$GARDEN_SYSTEM.$ARCH.qcow2" "$GARDEN_SYSTEM.seed.iso" "$GARDEN_SYSTEM.user-data" "$GARDEN_SYSTEM.meta-data" + - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" prepare_image + cache: + # Cache the base image (pre-customization). + - key: image-garden-base-${GARDEN_SYSTEM}.${ARCH} + policy: $CACHE_POLICY + when: always + paths: + - $GARDEN_DL_DIR + # Those are never mutated so they are safe to share. + - efi-code.*.img + - efi-vars.*.img + # Cache the customized system. This cache depends on .image-garden.mk file + # so that any customization updates are immediately acted upon. + - key: + prefix: image-garden-custom-${GARDEN_SYSTEM}.${ARCH}- + files: + - .image-garden.mk + policy: $CACHE_POLICY + when: always + paths: + - $GARDEN_SYSTEM.* + - $GARDEN_SYSTEM.seed.iso + - $GARDEN_SYSTEM.meta-data + - $GARDEN_SYSTEM.user-data + +# This job builds and caches the image that the job below looks at. +image-ubuntu-cloud-24.04-x86_64: + extends: .image-garden-x86_64 + variables: + GARDEN_SYSTEM: ubuntu-cloud-24.04 + needs: [] + dependencies: [] + rules: + - if: $CI_COMMIT_TAG + - if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH + changes: + paths: + - .image-garden.mk + - .gitlab-ci.yml + compare_to: "refs/heads/master" + +.spread-x86_64: + extends: .image-garden-x86_64 + variables: + # GitLab project identifier of zygoon/spread-dist can be seen on + # https://gitlab.com/zygoon/spread-dist, under the three-dot menu on + # top-right. + SPREAD_GITLAB_PROJECT_ID: "65375371" + # Git revision of spread to install. + # This must have been built via spread-dist. + # TODO: switch to upstream 1.0 release when available. + SPREAD_REV: 413817eda7bec07a3885e0717c178b965f8924e1 + # Run all the tasks for a given system. + SPREAD_ARGS: "garden:$GARDEN_SYSTEM:" + SPREAD_GOARCH: amd64 + before_script: + # Prepare the image in dry-run mode. This helps in debugging cache misses + # when files are not cached correctly by the runner, causing the build section + # below to always do hevy-duty work. + - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" prepare_image_dry_run "Prepare image (dry run)" + - image-garden make --dry-run --debug "$GARDEN_SYSTEM.$ARCH.run" "$GARDEN_SYSTEM.$ARCH.qcow2" "$GARDEN_SYSTEM.seed.iso" "$GARDEN_SYSTEM.user-data" "$GARDEN_SYSTEM.meta-data" + - stat .image-garden.mk "$GARDEN_SYSTEM".* || true + - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" prepare_image_dry_run + # Install the selected revision of spread. + - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" install_spread "Installing spread..." + # Install pre-built spread from https://gitlab.com/zygoon/spread-dist generic package repository. + - | + curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --location --output spread "${CI_API_V4_URL}/projects/${SPREAD_GITLAB_PROJECT_ID}/packages/generic/spread/${SPREAD_REV}/spread.${SPREAD_GOARCH}" + - chmod +x spread + - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" install_spread + script: + - printf '\e[0K%s:%s:%s\r\e[0K%s\n' section_start "$(date +%s)" run_spread "Running spread for $GARDEN_SYSTEM..." + # TODO: transform to inject ^...$ to properly select jobs to run. + - mkdir -p spread-logs spread-artifacts + - ./spread -list $SPREAD_ARGS | + split --number=l/"${CI_NODE_INDEX:-1}"/"${CI_NODE_TOTAL:-1}" | + xargs --verbose ./spread -v -artifacts ./spread-artifacts -v | tee spread-logs/"$GARDEN_SYSTEM".log + - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" run_spread + artifacts: + paths: + - spread-logs + - spread-artifacts + when: always + +spread-ubuntu-cloud-24.04-x86_64: + extends: .spread-x86_64 + variables: + GARDEN_SYSTEM: ubuntu-cloud-24.04 + SPREAD_ARGS: garden:$GARDEN_SYSTEM:tests/regression/ garden:$GARDEN_SYSTEM:tests/profiles/ + CACHE_POLICY: pull + dependencies: [] + needs: + - job: image-ubuntu-cloud-24.04-x86_64 + optional: true + parallel: 4 diff --git a/spread.yaml b/spread.yaml index 586d61d31..ee1ad81ed 100644 --- a/spread.yaml +++ b/spread.yaml @@ -66,7 +66,6 @@ backends: - ubuntu-cloud-24.04: username: ubuntu password: ubuntu - workers: 4 manual: true - ubuntu-cloud-24.10: username: ubuntu