test: refractor integration tests.

cmd/aa-test/main.go

@@ -8,11 +8,11 @@ import (
 	"flag"
 	"fmt"
 	"os"
-	"path/filepath"
 	"strings"
 
 	"github.com/arduino/go-paths-helper"
-	intg "github.com/roddhjav/apparmor.d/pkg/integration"
+	"github.com/roddhjav/apparmor.d/pkg/aa"
+	"github.com/roddhjav/apparmor.d/pkg/integration"
 	"github.com/roddhjav/apparmor.d/pkg/logging"
 )
 
@@ -22,10 +22,10 @@ const usage = `aa-test [-h] [--bootstrap | --run | --list]
 
 Options:
     -h, --help         Show this help message and exit.
-    -b, --bootstrap    Bootstrap test scenarios using tldr pages.
+    -b, --bootstrap    Bootstrap tests using tldr pages.
     -r, --run          Run a predefined list of tests.
     -l, --list         List the configured tests.
-    -f, --file FILE    Set a scenario test file. Default: tests/scenarios.yml
+    -f, --file FILE    Set a tests file. Default: tests/tests.yml
 
 `
 
@@ -36,15 +36,14 @@ var (
 	bootstrap bool
 	run       bool
 	list      bool
-	path      string
+	cfg       Config
-	Cfg       Config
 )
 
 type Config struct {
 	TldrDir      *paths.Path // Default: tests/tldr
 	ScenariosDir *paths.Path // Default: tests
-	ScenariosFile *paths.Path // Default: tests/tldr.yml
+	TldrFile     *paths.Path // Default: tests/tldr.yml
-	ProfilesDir   *paths.Path // Default: /etc/apparmor.d
+	TestsFile    *paths.Path // Default: tests/tests.yml
 	Profiles     paths.PathList
 }
 
@@ -52,105 +51,102 @@ func NewConfig() Config {
 	cfg := Config{
 		TldrDir:      paths.New("tests/tldr"),
 		ScenariosDir: paths.New("tests/"),
-		ProfilesDir:  paths.New("/etc/apparmor.d"),
 		Profiles:     paths.PathList{},
 	}
-	cfg.ScenariosFile = cfg.ScenariosDir.Join("tldr.yml")
+	cfg.TldrFile = cfg.ScenariosDir.Join("tldr.yml")
+	cfg.TestsFile = cfg.ScenariosDir.Join("tests.yml")
 	return cfg
 }
 
 func init() {
-	Cfg = NewConfig()
+	cfg = NewConfig()
-	files, _ := Cfg.ProfilesDir.ReadDir(paths.FilterOutDirectories())
+	files, _ := aa.MagicRoot.ReadDir(paths.FilterOutDirectories())
 	for _, path := range files {
-		Cfg.Profiles.Add(path)
+		cfg.Profiles.Add(path)
 	}
 
 	flag.BoolVar(&help, "h", false, "Show this help message and exit.")
 	flag.BoolVar(&help, "help", false, "Show this help message and exit.")
-	flag.BoolVar(&bootstrap, "b", false, "Bootstrap test scenarios using tldr pages.")
+	flag.BoolVar(&bootstrap, "b", false, "Bootstrap tests using tldr pages.")
-	flag.BoolVar(&bootstrap, "bootstrap", false, "Bootstrap test scenarios using tldr pages.")
+	flag.BoolVar(&bootstrap, "bootstrap", false, "Bootstrap tests using tldr pages.")
 	flag.BoolVar(&run, "r", false, "Run a predefined list of tests.")
 	flag.BoolVar(&run, "run", false, "Run a predefined list of tests.")
-	flag.BoolVar(&list, "l", false, "List the test to run.")
+	flag.BoolVar(&list, "l", false, "List the tests to run.")
-	flag.BoolVar(&list, "list", false, "List the test to run.")
+	flag.BoolVar(&list, "list", false, "List the tests to run.")
-	flag.StringVar(&path, "f", defaultScenariosFile, "Set a scenario test file.")
-	flag.StringVar(&path, "file", defaultScenariosFile, "Set a scenario test file.")
 }
 
-func apparmorTestBootstrap() error {
+func testDownload() error {
-	tldr := intg.NewTldr(Cfg.TldrDir)
+	tldr := integration.NewTldr(cfg.TldrDir)
 	if err := tldr.Download(); err != nil {
 		return err
 	}
 
-	tSuite, err := tldr.Parse(Cfg.Profiles)
+	tSuite, err := tldr.Parse(cfg.Profiles)
 	if err != nil {
 		return err
 	}
 
 	// Default bootstraped scenarios file
-	if err := tSuite.Write(Cfg.ScenariosFile); err != nil {
+	if err := tSuite.Write(cfg.TldrFile); err != nil {
 		return err
 	}
-	logging.Bullet("Default scenarios saved: %s", Cfg.ScenariosFile)
+	logging.Bullet("Default scenarios saved: %s", cfg.TldrFile)
 
 	// Scenarios file with only profiled programs
-	tSuiteWithProfile := intg.NewTestSuite()
+	tSuiteWithProfile := integration.NewTestSuite()
-	for _, s := range tSuite.Scenarios {
+	for _, t := range tSuite.Tests {
-		if s.Profiled {
+		if t.Profiled {
-			tSuiteWithProfile.Scenarios = append(tSuiteWithProfile.Scenarios, s)
+			tSuiteWithProfile.Tests = append(tSuiteWithProfile.Tests, t)
 		}
 	}
-	scnPath := Cfg.ScenariosDir.Join(strings.TrimSuffix(path, filepath.Ext(path)) + ".new.yml")
+
-	if err := tSuiteWithProfile.Write(scnPath); err != nil {
+	testWithProfilePath := cfg.TldrFile.Parent().Join(
+		strings.TrimSuffix(cfg.TldrFile.Base(), cfg.TldrFile.Ext()) + ".new.yml")
+	if err := tSuiteWithProfile.Write(testWithProfilePath); err != nil {
 		return err
 	}
-	logging.Bullet("Scenarios with profile saved: %s", scnPath)
+	logging.Bullet("Tests with profile saved: %s", testWithProfilePath)
 
-	logging.Bullet("Number of scenarios found %d", len(tSuite.Scenarios))
+	logging.Bullet("Number of tests found %d", len(tSuite.Tests))
-	logging.Bullet("Number of scenarios with profiles in apparmor.d %d", len(tSuiteWithProfile.Scenarios))
+	logging.Bullet("Number of tests with profiles in apparmor.d %d", len(tSuiteWithProfile.Tests))
 	return nil
 }
 
-func apparmorTestRun(dryRun bool) error {
+func testRun(dryRun bool) error {
-	// FIXME: Safety settings. For default scenario set with sudo enabled the apparmor.d folder gets removed.
+	// Warning: There is no guarantee that the tests are not destructive
-	dryRun = true
 	if dryRun {
 		logging.Step("List tests")
 	} else {
 		logging.Step("Run tests")
 	}
 
-	tSuite := intg.NewTestSuite()
+	tSuite := integration.NewTestSuite()
-	scnPath := Cfg.ScenariosDir.Join(path)
+	if err := tSuite.ReadScenarios(cfg.TestsFile); err != nil {
-	if err := tSuite.ReadScenarios(scnPath); err != nil {
 		return err
 	}
-	cfgPath := Cfg.ScenariosDir.Join("integration.yml")
+	cfgPath := cfg.ScenariosDir.Join("integration.yml")
 	if err := tSuite.ReadSettings(cfgPath); err != nil {
 		return err
 	}
-	intg.Arguments = tSuite.Arguments
+	integration.Arguments = tSuite.Arguments
-	intg.Ignore = tSuite.Ignore
+	integration.Ignore = tSuite.Ignore
-
+	nbCmd := 0
 	nbTest := 0
-	nbScn := 0
+	for _, test := range tSuite.Tests {
-	for _, scn := range tSuite.Scenarios {
+		ran, nb, err := test.Run(dryRun)
-		ran, nb, err := scn.Run(dryRun)
+		nbTest += ran
-		nbScn += ran
+		nbCmd += nb
-		nbTest += nb
 		if err != nil {
 			return err
 		}
 	}
 
 	if dryRun {
-		logging.Bullet("Number of scenarios to run %d", nbScn)
 		logging.Bullet("Number of tests to run %d", nbTest)
+		logging.Bullet("Number of test commands to run %d", nbCmd)
 	} else {
-		logging.Success("Number of scenarios ran %d", nbScn)
+		logging.Success("Number of tests ran %d", nbTest)
-		logging.Success("Number of tests to ran %d", nbTest)
+		logging.Success("Number of test command to ran %d", nbCmd)
 	}
 	return nil
 }
@@ -166,9 +162,9 @@ func main() {
 	var err error
 	if bootstrap {
 		logging.Step("Bootstraping tests")
-		err = apparmorTestBootstrap()
+		err = testDownload()
 	} else if run || list {
-		err = apparmorTestRun(list)
+		err = testRun(list)
 	} else {
 		flag.Usage()
 		os.Exit(1)

pkg/integration/scenario.go

@@ -21,84 +21,84 @@ import (
 	"golang.org/x/exp/slices"
 )
 
-// Scenario represents of a list of tests for a given program
+// Test represents of a list of tests for a given program
-type (
+type Test struct {
-	Scenario struct {
 	Name         string            `yaml:"name"`
 	Profiled     bool              `yaml:"profiled"`  // The program is profiled in apparmor.d
 	Root         bool              `yaml:"root"`      // Run the test as user or as root
 	Dependencies []string          `yaml:"require"`   // Packages required for the tests to run "$(pacman -Qqo Scenario.Name)"
-		Arguments    map[string]string `yaml:"arguments"` // Arguments to pass to the program. Sepicific to this scenario
+	Arguments    map[string]string `yaml:"arguments"` // Arguments to pass to the program, specific to this scenario
-		Tests        []Test            `yaml:"tests"`
+	Commands     []Command         `yaml:"tests"`
 }
-	Test struct {
+
+// Command is a command line to run as part of a test
+type Command struct {
 	Description string   `yaml:"dsc"`
-		Command     string   `yaml:"cmd"`
+	Cmd         string   `yaml:"cmd"`
 	Stdin       []string `yaml:"stdin"`
 }
-)
 
-func NewScenario() *Scenario {
+func NewTest() *Test {
-	return &Scenario{
+	return &Test{
 		Name:         "",
 		Profiled:     false,
 		Root:         false,
 		Dependencies: []string{},
 		Arguments:    map[string]string{},
-		Tests:        []Test{},
+		Commands:     []Command{},
 	}
 }
 
 // HasProfile returns true if the program in the scenario is profiled in apparmor.d
-func (s *Scenario) hasProfile(profiles paths.PathList) bool {
+func (t *Test) hasProfile(profiles paths.PathList) bool {
 	for _, path := range profiles {
-		if s.Name == path.Base() {
+		if t.Name == path.Base() {
 			return true
 		}
 	}
 	return false
 }
 
-func (s *Scenario) installed() bool {
+func (t *Test) installed() bool {
-	if _, err := exec.LookPath(s.Name); err != nil {
+	if _, err := exec.LookPath(t.Name); err != nil {
 		return false
 	}
 	return true
 }
 
-func (s *Scenario) resolve(in string) string {
+func (t *Test) resolve(in string) string {
 	res := in
-	for key, value := range s.Arguments {
+	for key, value := range t.Arguments {
 		res = strings.ReplaceAll(res, "{{ "+key+" }}", value)
 	}
 	return res
 }
 
 // mergeArguments merge the arguments of the scenario with the global arguments
-// Scenarios arguments have priority over global arguments
+// Test arguments have priority over global arguments
-func (s *Scenario) mergeArguments(args map[string]string) {
+func (t *Test) mergeArguments(args map[string]string) {
 	for key, value := range args {
-		s.Arguments[key] = value
+		t.Arguments[key] = value
 	}
 }
 
 // Run the scenarios tests
-func (s *Scenario) Run(dryRun bool) (ran int, nb int, err error) {
+func (t *Test) Run(dryRun bool) (ran int, nb int, err error) {
 	nb = 0
-	if s.Profiled && s.installed() {
+	if t.Profiled && t.installed() {
-		if slices.Contains(Ignore, s.Name) {
+		if slices.Contains(Ignore, t.Name) {
 			return 0, nb, err
 		}
-		logging.Step("%s", s.Name)
+		logging.Step("%s", t.Name)
-		s.mergeArguments(Arguments)
+		t.mergeArguments(Arguments)
-		for _, test := range s.Tests {
+		for _, test := range t.Commands {
-			cmd := s.resolve(test.Command)
+			cmd := t.resolve(test.Cmd)
 			if !strings.Contains(cmd, "{{") {
 				nb++
 				if dryRun {
 					logging.Bullet(cmd)
 				} else {
-					cmdErr := s.run(cmd, strings.Join(test.Stdin, "\n"))
+					cmdErr := t.run(cmd, strings.Join(test.Stdin, "\n"))
 					if cmdErr != nil {
 						// TODO: log the error
 						logging.Error("%v", cmdErr)
@@ -113,12 +113,12 @@ func (s *Scenario) Run(dryRun bool) (ran int, nb int, err error) {
 	return 0, nb, err
 }
 
-func (s *Scenario) run(cmdline string, in string) error {
+func (t *Test) run(cmdline string, in string) error {
 	// Running the command in a shell ensure it does not run confined under the sudo profile.
 	// The shell is run unconfined and therefore the cmdline can be confined without no-new-privs issue.
 	sufix := " &" // TODO: we need a goroutine here
 	cmd := exec.Command("sh", "-c", cmdline+sufix)
-	if s.Root {
+	if t.Root {
 		cmd = exec.Command("sudo", "sh", "-c", cmdline+sufix)
 	}
 	cmd.Stdin = strings.NewReader(in)

pkg/integration/suite.go

@@ -5,24 +5,23 @@
 package integration
 
 import (
-	"strings"
-
 	"github.com/arduino/go-paths-helper"
 	"github.com/roddhjav/apparmor.d/pkg/logs"
+	"github.com/roddhjav/apparmor.d/pkg/util"
 	"gopkg.in/yaml.v2"
 )
 
 // TestSuite is the apparmod.d integration tests to run
 type TestSuite struct {
-	Scenarios []Scenario        // List of scenarios to run
+	Tests     []Test            // List of tests to run
-	Ignore    []string          // Do not run some scenarios
+	Ignore    []string          // Do not run some tests
-	Arguments map[string]string // Common arguments used across all scenarios
+	Arguments map[string]string // Common arguments used across all tests
 }
 
 // NewScenarios returns a new list of scenarios
 func NewTestSuite() *TestSuite {
 	return &TestSuite{
-		Scenarios: []Scenario{},
+		Tests:     []Test{},
 		Ignore:    []string{},
 		Arguments: map[string]string{},
 	}
@@ -30,7 +29,7 @@ func NewTestSuite() *TestSuite {
 
 // Write export the list of scenarios to a file
 func (t *TestSuite) Write(path *paths.Path) error {
-	jsonString, err := yaml.Marshal(&t.Scenarios)
+	jsonString, err := yaml.Marshal(&t.Tests)
 	if err != nil {
 		return err
 	}
@@ -44,7 +43,15 @@ func (t *TestSuite) Write(path *paths.Path) error {
 
 	// Cleanup a bit
 	res := string(jsonString)
-	res = strings.Replace(res, "- name:", "\n- name:", -1)
+	regClean := util.ToRegexRepl([]string{
+		"- name:", "\n- name:",
+		`(?m)^.*stdin: \[\].*$`, ``,
+		`{{`, `{{ `,
+		`}}`, ` }}`,
+	})
+	for _, aa := range regClean {
+		res = aa.Regex.ReplaceAllLiteralString(res, aa.Repl)
+	}
 	_, err = file.WriteString("---\n" + res)
 	return err
 }
@@ -52,7 +59,7 @@ func (t *TestSuite) Write(path *paths.Path) error {
 // ReadScenarios import the scenarios from a file
 func (t *TestSuite) ReadScenarios(path *paths.Path) error {
 	content, _ := path.ReadFile()
-	return yaml.Unmarshal(content, &t.Scenarios)
+	return yaml.Unmarshal(content, &t.Tests)
 }
 
 // ReadSettings import the common argument and ignore list from a file

pkg/integration/tldr.go

@@ -54,10 +54,7 @@ func (t Tldr) Download() error {
 	}
 
 	pages := []string{"tldr-main/pages/linux", "tldr-main/pages/common"}
-	if err := util.ExtratTo(gzPath, t.Dir, pages); err != nil {
+	return util.ExtratTo(gzPath, t.Dir, pages)
-		return err
-	}
-	return nil
 }
 
 // Parse the tldr pages and return a list of scenarios
@@ -70,31 +67,31 @@ func (t Tldr) Parse(profiles paths.PathList) (*TestSuite, error) {
 			return nil, err
 		}
 		raw := string(content)
-		scenario := &Scenario{
+		t := &Test{
 			Name:      strings.TrimSuffix(path.Base(), ".md"),
 			Profiled:  false,
 			Root:      false,
 			Arguments: map[string]string{},
-			Tests:     []Test{},
+			Commands:  []Command{},
 		}
-		scenario.Profiled = scenario.hasProfile(profiles)
+		t.Profiled = t.hasProfile(profiles)
 		if strings.Contains(raw, "sudo") {
-			scenario.Root = true
+			t.Root = true
 		}
 		rawTests := strings.Split(raw, "\n-")[1:]
 		for _, test := range rawTests {
 			res := strings.Split(test, "\n")
 			dsc := strings.ReplaceAll(strings.Trim(res[0], " "), ":", "")
 			cmd := strings.Trim(strings.Trim(res[2], "`"), " ")
-			if scenario.Root {
+			if t.Root {
 				cmd = strings.ReplaceAll(cmd, "sudo ", "")
 			}
-			scenario.Tests = append(scenario.Tests, Test{
+			t.Commands = append(t.Commands, Command{
 				Description: dsc,
-				Command:     cmd,
+				Cmd:         cmd,
 			})
 		}
-		testSuite.Scenarios = append(testSuite.Scenarios, *scenario)
+		testSuite.Tests = append(testSuite.Tests, *t)
 	}
 	return testSuite, nil
 }