tests: rewrite the way to generate integration tests.

This commit is contained in:
Alexandre Pujol 2024-10-19 22:50:09 +01:00
parent f079792aee
commit c59086311b
Failed to generate hash of commit
7 changed files with 217 additions and 484 deletions

1
go.mod
View file

@ -4,7 +4,6 @@ go 1.21
require (
github.com/stretchr/testify v1.9.0
gopkg.in/yaml.v3 v3.0.1
)
require (

View file

@ -8,171 +8,76 @@ import (
"flag"
"fmt"
"os"
"os/exec"
"strings"
"github.com/roddhjav/apparmor.d/pkg/aa"
"github.com/roddhjav/apparmor.d/pkg/logging"
"github.com/roddhjav/apparmor.d/pkg/paths"
"github.com/roddhjav/apparmor.d/pkg/prebuild"
"github.com/roddhjav/apparmor.d/tests/integration"
)
const usage = `aa-test [-h] [--bootstrap | --run | --list]
const usage = `aa-test [-h] --bootstrap
Integration tests manager tool for apparmor.d
Options:
-h, --help Show this help message and exit.
-b, --bootstrap Bootstrap tests using tldr pages.
-r, --run Run a predefined list of tests.
-l, --list List the configured tests.
-f, --file FILE Set a tests file. Default: tests/tests.yml
-d, --deps Install tests dependencies.
-D, --dryrun Do not do the action, list it.
-b, --bootstrap Download tests using tldr pages and generate Bats tests.
`
var (
help bool
bootstrap bool
run bool
list bool
deps bool
dryRun bool
cfg Config
)
type Config struct {
TldrDir *paths.Path // Default: tests/tldr
ScenariosDir *paths.Path // Default: tests
TldrFile *paths.Path // Default: tests/tldr.yml
TestsFile *paths.Path // Default: tests/tests.yml
SettingsFile *paths.Path // Default: tests/settings.yml
Profiles paths.PathList // List of profiles
}
func NewConfig() Config {
cfg := Config{
TldrDir: paths.New("tests/tldr"),
ScenariosDir: paths.New("tests/"),
Profiles: paths.PathList{},
}
cfg.TldrFile = cfg.ScenariosDir.Join("tldr.yml")
cfg.TestsFile = cfg.ScenariosDir.Join("tests.yml")
cfg.SettingsFile = cfg.ScenariosDir.Join("settings.yml")
return cfg
}
func LoadTestSuite() (*integration.TestSuite, error) {
tSuite := integration.NewTestSuite()
if err := tSuite.ReadTests(cfg.TestsFile); err != nil {
return tSuite, err
}
if err := tSuite.ReadSettings(cfg.SettingsFile); err != nil {
return tSuite, err
}
return tSuite, nil
}
func init() {
cfg = NewConfig()
files, _ := aa.MagicRoot.ReadDir(paths.FilterOutDirectories())
for _, path := range files {
cfg.Profiles.Add(path)
}
flag.BoolVar(&help, "h", false, "Show this help message and exit.")
flag.BoolVar(&help, "help", false, "Show this help message and exit.")
flag.BoolVar(&bootstrap, "b", false, "Bootstrap tests using tldr pages.")
flag.BoolVar(&bootstrap, "bootstrap", false, "Bootstrap tests using tldr pages.")
flag.BoolVar(&run, "r", false, "Run a predefined list of tests.")
flag.BoolVar(&run, "run", false, "Run a predefined list of tests.")
flag.BoolVar(&list, "l", false, "List the tests to run.")
flag.BoolVar(&list, "list", false, "List the tests to run.")
flag.BoolVar(&deps, "d", false, "Install tests dependencies.")
flag.BoolVar(&deps, "deps", false, "Install tests dependencies.")
flag.BoolVar(&dryRun, "D", false, "Do not do the action, list it.")
flag.BoolVar(&dryRun, "dryrun", false, "Do not do the action, list it.")
flag.BoolVar(&bootstrap, "b", false, "Download tests using tldr pages and generate Bats tests.")
flag.BoolVar(&bootstrap, "bootstrap", false, "Download tests using tldr pages and generate Bats tests.")
}
func testDownload() error {
tldr := integration.NewTldr(cfg.TldrDir)
type Config struct {
TestsDir *paths.Path // Default: tests
TldrDir *paths.Path // Default: tests/tldr
TldrFile *paths.Path // Default: tests/tldr.yml
TestsFile *paths.Path // Default: tests/tests.yml
BatsDir *paths.Path // Default: tests/bats
}
func NewConfig() *Config {
testsDir := paths.New("tests")
cfg := Config{
TestsDir: testsDir,
TldrDir: testsDir.Join("tldr"),
TldrFile: testsDir.Join("tldr.yml"),
TestsFile: testsDir.Join("tldr.yml"),
BatsDir: testsDir.Join("bats_dirty"),
}
return &cfg
}
func run() error {
logging.Step("Bootstraping tests")
cfg := NewConfig()
tldr := NewTldr(cfg.TldrDir)
if err := tldr.Download(); err != nil {
return err
}
tSuite, err := tldr.Parse()
tests, err := tldr.Parse()
if err != nil {
return err
}
tests = tests.Filter()
// Default bootstraped scenarios file
if err := tSuite.Write(cfg.TldrFile); err != nil {
return err
}
logging.Bullet("Default scenarios saved: %s", cfg.TldrFile)
logging.Bullet("Number of tests found %d", len(tSuite.Tests))
return nil
}
func testDeps(dryRun bool) error {
tSuite, err := LoadTestSuite()
if err != nil {
return nil
}
deps := tSuite.GetDependencies()
switch prebuild.Distribution {
case "arch":
arg := []string{"pacman", "-Sy", "--noconfirm"}
arg = append(arg, deps...)
cmd := exec.Command("sudo", arg...)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if dryRun {
fmt.Println(strings.Join(cmd.Args, " "))
} else {
return cmd.Run()
}
default:
}
return nil
}
func testRun(dryRun bool) error {
// Warning: There is no guarantee that the tests are not destructive
if dryRun {
logging.Step("List tests")
} else {
logging.Step("Run tests")
}
tSuite, err := LoadTestSuite()
if err != nil {
return nil
}
integration.Arguments = tSuite.Arguments
integration.Ignore = tSuite.Ignore
integration.Profiles = cfg.Profiles
nbCmd := 0
nbTest := 0
for _, test := range tSuite.Tests {
ran, nb, err := test.Run(dryRun)
nbTest += ran
nbCmd += nb
if err != nil {
for _, test := range tests {
if err := test.Write(cfg.BatsDir); err != nil {
return err
}
}
if dryRun {
logging.Bullet("Number of tests to run %d", nbTest)
logging.Bullet("Number of test commands to run %d", nbCmd)
} else {
logging.Success("Number of tests ran %d", nbTest)
logging.Success("Number of test command to ran %d", nbCmd)
}
logging.Bullet("Bats tests directory: %s", cfg.BatsDir)
logging.Bullet("Number of tests found %d", len(tests))
return nil
}
@ -184,18 +89,12 @@ func main() {
os.Exit(0)
}
var err error
if bootstrap {
logging.Step("Bootstraping tests")
err = testDownload()
} else if run || list {
err = testRun(list)
} else if deps {
err = testDeps(dryRun)
} else {
if !bootstrap {
flag.Usage()
os.Exit(1)
}
err := run()
if err != nil {
logging.Fatal("%s", err.Error())
}

111
tests/cmd/tests.go Normal file
View file

@ -0,0 +1,111 @@
// apparmor.d - Full set of apparmor profiles
// Copyright (C) 2023-2024 Alexandre Pujol <alexandre@pujol.io>
// SPDX-License-Identifier: GPL-2.0-only
package main
import (
"html/template"
"os/exec"
"slices"
"strings"
"github.com/roddhjav/apparmor.d/pkg/aa"
"github.com/roddhjav/apparmor.d/pkg/paths"
)
const tmplTest = `#!/usr/bin/env bats
# apparmor.d - Full set of apparmor profiles
# Copyright (C) 2024 Alexandre Pujol <alexandre@pujol.io>
# SPDX-License-Identifier: GPL-2.0-only
{{ $name := .Name -}}
{{ range .Commands }}
# bats test_tags={{ $name }}
@test "{{ $name }}: {{ .Description }}" {
{{ .Cmd }}
}
{{ end }}
`
var (
Profiles = getProfiles() // List of profiles in apparmor.d
tmpl = template.Must(template.New("bats").Parse(tmplTest))
)
type Tests []Test
// Filter returns a new list of tests with only the ones that have a profile
func (t Tests) Filter() Tests {
for i := len(t) - 1; i >= 0; i-- {
if !t[i].HasProfile() {
t = slices.Delete(t, i, i+1)
}
}
return t
}
// Test represents of a list of tests for a given program
type Test struct {
Name string
Commands []Command
}
// Command is a command line to run as part of a test
type Command struct {
Description string
Cmd string
}
func NewTest() *Test {
return &Test{
Name: "",
Commands: []Command{},
}
}
// HasProfile returns true if the program in the scenario is profiled in apparmor.d
func (t *Test) HasProfile() bool {
return slices.Contains(Profiles, t.Name)
}
// IsInstalled returns true if the program in the scenario is installed on the system
func (t *Test) IsInstalled() bool {
if _, err := exec.LookPath(t.Name); err != nil {
return false
}
return true
}
func (t Test) Write(dir *paths.Path) error {
if !t.HasProfile() {
return nil
}
path := dir.Join(t.Name + ".bats")
content := renderBatsFile(t)
if err := path.WriteFile([]byte(content)); err != nil {
return err
}
return nil
}
func renderBatsFile(data any) string {
var res strings.Builder
err := tmpl.Execute(&res, data)
if err != nil {
panic(err)
}
return res.String()
}
func getProfiles() []string {
p := []string{}
files, err := aa.MagicRoot.ReadDir(paths.FilterOutDirectories())
if err != nil {
panic(err)
}
for _, path := range files {
p = append(p, path.Base())
}
return p
}

View file

@ -2,12 +2,15 @@
// Copyright (C) 2023-2024 Alexandre Pujol <alexandre@pujol.io>
// SPDX-License-Identifier: GPL-2.0-only
package integration
package main
import (
"archive/tar"
"compress/gzip"
"fmt"
"io"
"net/http"
"path/filepath"
"strings"
"github.com/roddhjav/apparmor.d/pkg/paths"
@ -51,9 +54,9 @@ func (t Tldr) Download() error {
return extratTo(gzPath, t.Dir, pages)
}
// Parse the tldr pages and return a list of scenarios
func (t Tldr) Parse() (*TestSuite, error) {
testSuite := NewTestSuite()
// Parse the tldr pages and return a list of tests
func (t Tldr) Parse() (Tests, error) {
tests := make(Tests, 0)
files, _ := t.Dir.ReadDirRecursiveFiltered(nil, paths.FilterOutDirectories())
for _, path := range files {
content, err := path.ReadFile()
@ -61,29 +64,77 @@ func (t Tldr) Parse() (*TestSuite, error) {
return nil, err
}
raw := string(content)
t := &Test{
t := Test{
Name: strings.TrimSuffix(path.Base(), ".md"),
Root: false,
Arguments: map[string]string{},
Commands: []Command{},
}
if strings.Contains(raw, "sudo") {
t.Root = true
}
rawTests := strings.Split(raw, "\n-")[1:]
for _, test := range rawTests {
res := strings.Split(test, "\n")
dsc := strings.ReplaceAll(strings.Trim(res[0], " "), ":", "")
cmd := strings.Trim(strings.Trim(res[2], "`"), " ")
if t.Root {
cmd = strings.ReplaceAll(cmd, "sudo ", "")
}
t.Commands = append(t.Commands, Command{
Description: dsc,
Cmd: cmd,
})
}
testSuite.Tests = append(testSuite.Tests, *t)
tests = append(tests, t)
}
return testSuite, nil
return tests, nil
}
// Either or not to extract the file
func toExtrat(name string, subfolders []string) bool {
for _, subfolder := range subfolders {
if strings.HasPrefix(name, subfolder) {
return true
}
}
return false
}
// Extract part of an archive to a destination directory
func extratTo(src *paths.Path, dst *paths.Path, subfolders []string) error {
gzIn, err := src.Open()
if err != nil {
return fmt.Errorf("opening %s: %w", src, err)
}
defer gzIn.Close()
in, err := gzip.NewReader(gzIn)
if err != nil {
return fmt.Errorf("decoding %s: %w", src, err)
}
defer in.Close()
if err := dst.MkdirAll(); err != nil {
return fmt.Errorf("creating %s: %w", src, err)
}
tarIn := tar.NewReader(in)
for {
header, err := tarIn.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
if header.Typeflag == tar.TypeReg {
if !toExtrat(header.Name, subfolders) {
continue
}
path := dst.Join(filepath.Base(header.Name))
file, err := path.Create()
if err != nil {
return fmt.Errorf("creating %s: %w", file.Name(), err)
}
if _, err := io.Copy(file, tarIn); err != nil {
return fmt.Errorf("extracting %s: %w", file.Name(), err)
}
file.Close()
}
}
return nil
}

View file

@ -1,72 +0,0 @@
// apparmor.d - Full set of apparmor profiles
// Copyright (C) 2023-2024 Alexandre Pujol <alexandre@pujol.io>
// SPDX-License-Identifier: GPL-2.0-only
package integration
import (
"archive/tar"
"compress/gzip"
"fmt"
"io"
"path/filepath"
"strings"
"github.com/roddhjav/apparmor.d/pkg/paths"
)
// Either or not to extract the file
func toExtrat(name string, subfolders []string) bool {
for _, subfolder := range subfolders {
if strings.HasPrefix(name, subfolder) {
return true
}
}
return false
}
// Extract part of an archive to a destination directory
func extratTo(src *paths.Path, dst *paths.Path, subfolders []string) error {
gzIn, err := src.Open()
if err != nil {
return fmt.Errorf("opening %s: %w", src, err)
}
defer gzIn.Close()
in, err := gzip.NewReader(gzIn)
if err != nil {
return fmt.Errorf("decoding %s: %w", src, err)
}
defer in.Close()
if err := dst.MkdirAll(); err != nil {
return fmt.Errorf("creating %s: %w", src, err)
}
tarIn := tar.NewReader(in)
for {
header, err := tarIn.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
if header.Typeflag == tar.TypeReg {
if !toExtrat(header.Name, subfolders) {
continue
}
path := dst.Join(filepath.Base(header.Name))
file, err := path.Create()
if err != nil {
return fmt.Errorf("creating %s: %w", file.Name(), err)
}
if _, err := io.Copy(file, tarIn); err != nil {
return fmt.Errorf("extracting %s: %w", file.Name(), err)
}
file.Close()
}
}
return nil
}

View file

@ -1,141 +0,0 @@
// apparmor.d - Full set of apparmor profiles
// Copyright (C) 2023-2024 Alexandre Pujol <alexandre@pujol.io>
// SPDX-License-Identifier: GPL-2.0-only
// TODO:
// - Finish templating
// - Provide a large selection of resources: files, disks, http server... for automatic test on them
// - Expand support for interactive program (stdin and Control-D)
// - Properlly log the test result
// - Dbus integration
package integration
import (
"bytes"
"fmt"
"io"
"os/exec"
"strings"
"github.com/roddhjav/apparmor.d/pkg/logging"
"github.com/roddhjav/apparmor.d/pkg/paths"
)
var (
Ignore []string // Do not run some scenarios
Arguments map[string]string // Common arguments used across all scenarios
Profiles paths.PathList // List of profiles in apparmor.d
)
// Test represents of a list of tests for a given program
type Test struct {
Name string `yaml:"name"`
Root bool `yaml:"root"` // Run the test as user or as root
Dependencies []string `yaml:"require"` // Packages required for the tests to run "$(pacman -Qqo Scenario.Name)"
Arguments map[string]string `yaml:"arguments"` // Arguments to pass to the program, specific to this scenario
Commands []Command `yaml:"tests"`
}
// Command is a command line to run as part of a test
type Command struct {
Description string `yaml:"dsc"`
Cmd string `yaml:"cmd"`
Stdin []string `yaml:"stdin"`
}
func NewTest() *Test {
return &Test{
Name: "",
Root: false,
Dependencies: []string{},
Arguments: map[string]string{},
Commands: []Command{},
}
}
// HasProfile returns true if the program in the scenario is profiled in apparmor.d
func (t *Test) HasProfile() bool {
for _, path := range Profiles {
if t.Name == path.Base() {
return true
}
}
return false
}
// IsInstalled returns true if the program in the scenario is installed on the system
func (t *Test) IsInstalled() bool {
if _, err := exec.LookPath(t.Name); err != nil {
return false
}
return true
}
func (t *Test) resolve(in string) string {
res := in
for key, value := range t.Arguments {
res = strings.ReplaceAll(res, "{{ "+key+" }}", value)
}
return res
}
// mergeArguments merge the arguments of the scenario with the global arguments
// Test arguments have priority over global arguments
func (t *Test) mergeArguments(args map[string]string) {
if len(t.Arguments) == 0 {
t.Arguments = map[string]string{}
}
for key, value := range args {
t.Arguments[key] = value
}
}
// Run the scenarios tests
func (t *Test) Run(dryRun bool) (ran int, nb int, err error) {
nb = 0
if t.HasProfile() && t.IsInstalled() {
logging.Step("%s", t.Name)
t.mergeArguments(Arguments)
for _, test := range t.Commands {
cmd := t.resolve(test.Cmd)
if !strings.Contains(cmd, "{{") {
nb++
if dryRun {
logging.Bullet("%s", cmd)
} else {
cmdErr := t.run(cmd, strings.Join(test.Stdin, "\n"))
if cmdErr != nil {
logging.Error("%v", cmdErr)
} else {
logging.Success("%s", cmd)
}
}
}
}
return 1, nb, err
}
return 0, nb, err
}
func (t *Test) run(cmdline string, in string) error {
var testErr bytes.Buffer
// Running the command in a shell ensure it does not run confined under the sudo profile.
// The shell is run unconfined and therefore the cmdline can be confined without no-new-privs issue.
sufix := " &" // TODO: we need a goroutine here
cmd := exec.Command("sh", "-c", cmdline+sufix)
if t.Root {
cmd = exec.Command("sudo", "sh", "-c", cmdline+sufix)
}
stderr := io.MultiWriter(Stderr, &testErr)
cmd.Stdin = strings.NewReader(in)
cmd.Stdout = Stdout
cmd.Stderr = stderr
err := cmd.Run()
if testErr.Len() > 0 {
return fmt.Errorf("%s", testErr.String())
}
return err
}

View file

@ -1,114 +0,0 @@
// apparmor.d - Full set of apparmor profiles
// Copyright (C) 2023-2024 Alexandre Pujol <alexandre@pujol.io>
// SPDX-License-Identifier: GPL-2.0-only
package integration
import (
"os"
"github.com/roddhjav/apparmor.d/pkg/logs"
"github.com/roddhjav/apparmor.d/pkg/paths"
"github.com/roddhjav/apparmor.d/pkg/util"
"gopkg.in/yaml.v3"
)
var (
// Integration tests standard output
Stdout *os.File
// Integration tests standard error output
Stderr *os.File
stdoutPath = paths.New("tests/out.log")
stderrPath = paths.New("tests/err.log")
)
// TestSuite is the apparmod.d integration tests to run
type TestSuite struct {
Tests []Test // List of tests to run
Ignore []string // Do not run some tests
Arguments map[string]string // Common arguments used across all tests
}
// NewScenarios returns a new list of scenarios
func NewTestSuite() *TestSuite {
var err error
Stdout, err = stdoutPath.Create()
if err != nil {
panic(err)
}
Stderr, err = stderrPath.Create()
if err != nil {
panic(err)
}
return &TestSuite{
Tests: []Test{},
Ignore: []string{},
Arguments: map[string]string{},
}
}
// Write export the list of scenarios to a file
func (t *TestSuite) Write(path *paths.Path) error {
jsonString, err := yaml.Marshal(&t.Tests)
if err != nil {
return err
}
path = path.Clean()
file, err := path.Create()
if err != nil {
return err
}
defer file.Close()
// Cleanup a bit
res := string(jsonString)
regClean := util.ToRegexRepl([]string{
"- name:", "\n- name:",
`(?m)^.*stdin: \[\].*$`, ``,
`{{`, `{{ `,
`}}`, ` }}`,
})
res = regClean.Replace(res)
_, err = file.WriteString("---\n" + res)
return err
}
// ReadTests import the tests from a file
func (t *TestSuite) ReadTests(path *paths.Path) error {
content, _ := path.ReadFile()
return yaml.Unmarshal(content, &t.Tests)
}
// ReadSettings import the common argument and ignore list from a file
func (t *TestSuite) ReadSettings(path *paths.Path) error {
type temp struct {
Arguments map[string]string `yaml:"arguments"`
Ignore []string `yaml:"ignore"`
}
tmp := temp{}
content, _ := path.ReadFile()
if err := yaml.Unmarshal(content, &tmp); err != nil {
return err
}
t.Arguments = tmp.Arguments
t.Ignore = tmp.Ignore
return nil
}
// Results returns a sum up of the apparmor logs raised by the scenarios
func (t *TestSuite) Results() string {
file, _ := logs.GetAuditLogs(logs.LogFiles[0])
aaLogs := logs.New(file, "")
return aaLogs.String()
}
func (t *TestSuite) GetDependencies() []string {
res := []string{}
for _, test := range t.Tests {
res = append(res, test.Dependencies...)
}
return res
}