test(integration): initial version of integration tests manager

cmd/aa-test/main.go

@@ -0,0 +1,179 @@
+// apparmor.d - Full set of apparmor profiles
+// Copyright (C) 2023 Alexandre Pujol <alexandre@pujol.io>
+// SPDX-License-Identifier: GPL-2.0-only
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/arduino/go-paths-helper"
+	intg "github.com/roddhjav/apparmor.d/pkg/integration"
+	"github.com/roddhjav/apparmor.d/pkg/logging"
+)
+
+const usage = `aa-test [-h] [--bootstrap | --run | --list]
+
+    Integration tests manager tool for apparmor.d
+
+Options:
+    -h, --help         Show this help message and exit.
+    -b, --bootstrap    Bootstrap test scenarios using tldr pages.
+    -r, --run          Run a predefined list of tests.
+    -l, --list         List the configured tests.
+    -f, --file FILE    Set a scenario test file. Default: tests/scenarios.yml
+
+`
+
+const defaultScenariosFile = "scenarios.yml"
+
+var (
+	help      bool
+	bootstrap bool
+	run       bool
+	list      bool
+	path      string
+	Cfg       Config
+)
+
+type Config struct {
+	TldrDir       *paths.Path // Default: tests/tldr
+	ScenariosDir  *paths.Path // Default: tests
+	ScenariosFile *paths.Path // Default: tests/tldr.yml
+	ProfilesDir   *paths.Path // Default: /etc/apparmor.d
+	Profiles      paths.PathList
+}
+
+func NewConfig() Config {
+	cfg := Config{
+		TldrDir:      paths.New("tests/tldr"),
+		ScenariosDir: paths.New("tests/"),
+		ProfilesDir:  paths.New("/etc/apparmor.d"),
+		Profiles:     paths.PathList{},
+	}
+	cfg.ScenariosFile = cfg.ScenariosDir.Join("tldr.yml")
+	return cfg
+}
+
+func init() {
+	Cfg = NewConfig()
+	files, _ := Cfg.ProfilesDir.ReadDir(paths.FilterOutDirectories())
+	for _, path := range files {
+		Cfg.Profiles.Add(path)
+	}
+
+	flag.BoolVar(&help, "h", false, "Show this help message and exit.")
+	flag.BoolVar(&help, "help", false, "Show this help message and exit.")
+	flag.BoolVar(&bootstrap, "b", false, "Bootstrap test scenarios using tldr pages.")
+	flag.BoolVar(&bootstrap, "bootstrap", false, "Bootstrap test scenarios using tldr pages.")
+	flag.BoolVar(&run, "r", false, "Run a predefined list of tests.")
+	flag.BoolVar(&run, "run", false, "Run a predefined list of tests.")
+	flag.BoolVar(&list, "l", false, "List the test to run.")
+	flag.BoolVar(&list, "list", false, "List the test to run.")
+	flag.StringVar(&path, "f", defaultScenariosFile, "Set a scenario test file.")
+	flag.StringVar(&path, "file", defaultScenariosFile, "Set a scenario test file.")
+}
+
+func apparmorTestBootstrap() error {
+	tldr := intg.NewTldr(Cfg.TldrDir)
+	if err := tldr.Download(); err != nil {
+		return err
+	}
+
+	tSuite, err := tldr.Parse(Cfg.Profiles)
+	if err != nil {
+		return err
+	}
+
+	// Default bootstraped scenarios file
+	if err := tSuite.Write(Cfg.ScenariosFile); err != nil {
+		return err
+	}
+	logging.Bullet("Default scenarios saved: %s", Cfg.ScenariosFile)
+
+	// Scenarios file with only profiled programs
+	tSuiteWithProfile := intg.NewTestSuite()
+	for _, s := range tSuite.Scenarios {
+		if s.Profiled {
+			tSuiteWithProfile.Scenarios = append(tSuiteWithProfile.Scenarios, s)
+		}
+	}
+	scnPath := Cfg.ScenariosDir.Join(strings.TrimSuffix(path, filepath.Ext(path)) + ".new.yml")
+	if err := tSuiteWithProfile.Write(scnPath); err != nil {
+		return err
+	}
+	logging.Bullet("Scenarios with profile saved: %s", scnPath)
+
+	logging.Bullet("Number of scenarios found %d", len(tSuite.Scenarios))
+	logging.Bullet("Number of scenarios with profiles in apparmor.d %d", len(tSuiteWithProfile.Scenarios))
+	return nil
+}
+
+func apparmorTestRun(dryRun bool) error {
+	// FIXME: Safety settings. For default scenario set with sudo enabled the apparmor.d folder gets removed.
+	dryRun = true
+	if dryRun {
+		logging.Step("List tests")
+	} else {
+		logging.Step("Run tests")
+	}
+
+	tSuite := intg.NewTestSuite()
+	scnPath := Cfg.ScenariosDir.Join(path)
+	if err := tSuite.ReadScenarios(scnPath); err != nil {
+		return err
+	}
+	cfgPath := Cfg.ScenariosDir.Join("integration.yml")
+	if err := tSuite.ReadSettings(cfgPath); err != nil {
+		return err
+	}
+	intg.Arguments = tSuite.Arguments
+	intg.Ignore = tSuite.Ignore
+
+	nbTest := 0
+	nbScn := 0
+	for _, scn := range tSuite.Scenarios {
+		ran, nb, err := scn.Run(dryRun)
+		nbScn += ran
+		nbTest += nb
+		if err != nil {
+			return err
+		}
+	}
+
+	if dryRun {
+		logging.Bullet("Number of scenarios to run %d", nbScn)
+		logging.Bullet("Number of tests to run %d", nbTest)
+	} else {
+		logging.Success("Number of scenarios ran %d", nbScn)
+		logging.Success("Number of tests to ran %d", nbTest)
+	}
+	return nil
+}
+
+func main() {
+	flag.Usage = func() { fmt.Print(usage) }
+	flag.Parse()
+	if help {
+		flag.Usage()
+		os.Exit(0)
+	}
+
+	var err error
+	if bootstrap {
+		logging.Step("Bootstraping tests")
+		err = apparmorTestBootstrap()
+	} else if run || list {
+		err = apparmorTestRun(list)
+	} else {
+		flag.Usage()
+		os.Exit(1)
+	}
+	if err != nil {
+		logging.Fatal(err.Error())
+	}
+}

pkg/integration/scenario.go

@@ -0,0 +1,128 @@
+// apparmor.d - Full set of apparmor profiles
+// Copyright (C) 2023 Alexandre Pujol <alexandre@pujol.io>
+// SPDX-License-Identifier: GPL-2.0-only
+
+// TODO:
+// - Finish templating
+// - Provide a large selection of resources: files, disks, http server... for automatic test on them
+// - Expand support for interactive program (stdin and Control-D)
+// - Properlly log the test result
+// - Dbus integration
+
+package integration
+
+import (
+	"os"
+	"os/exec"
+	"strings"
+
+	"github.com/arduino/go-paths-helper"
+	"github.com/roddhjav/apparmor.d/pkg/logging"
+	"golang.org/x/exp/slices"
+)
+
+// Scenario represents of a list of tests for a given program
+type (
+	Scenario struct {
+		Name         string            `yaml:"name"`
+		Profiled     bool              `yaml:"profiled"`  // The program is profiled in apparmor.d
+		Root         bool              `yaml:"root"`      // Run the test as user or as root
+		Dependencies []string          `yaml:"require"`   // Packages required for the tests to run "$(pacman -Qqo Scenario.Name)"
+		Arguments    map[string]string `yaml:"arguments"` // Arguments to pass to the program. Sepicific to this scenario
+		Tests        []Test            `yaml:"tests"`
+	}
+	Test struct {
+		Description string   `yaml:"dsc"`
+		Command     string   `yaml:"cmd"`
+		Stdin       []string `yaml:"stdin"`
+	}
+)
+
+func NewScenario() *Scenario {
+	return &Scenario{
+		Name:         "",
+		Profiled:     false,
+		Root:         false,
+		Dependencies: []string{},
+		Arguments:    map[string]string{},
+		Tests:        []Test{},
+	}
+}
+
+// HasProfile returns true if the program in the scenario is profiled in apparmor.d
+func (s *Scenario) hasProfile(profiles paths.PathList) bool {
+	for _, path := range profiles {
+		if s.Name == path.Base() {
+			return true
+		}
+	}
+	return false
+}
+
+func (s *Scenario) installed() bool {
+	if _, err := exec.LookPath(s.Name); err != nil {
+		return false
+	}
+	return true
+}
+
+func (s *Scenario) resolve(in string) string {
+	res := in
+	for key, value := range s.Arguments {
+		res = strings.ReplaceAll(res, "{{"+key+"}}", value)
+	}
+	return res
+}
+
+// mergeArguments merge the arguments of the scenario with the global arguments
+// Scenarios arguments have priority over global arguments
+func (s *Scenario) mergeArguments(args map[string]string) {
+	for key, value := range args {
+		s.Arguments[key] = value
+	}
+}
+
+// Run the scenarios tests
+func (s *Scenario) Run(dryRun bool) (ran int, nb int, err error) {
+	nb = 0
+	if s.Profiled && s.installed() {
+		if slices.Contains(Ignore, s.Name) {
+			return 0, nb, err
+		}
+		logging.Step("%s", s.Name)
+		s.mergeArguments(Arguments)
+		for _, test := range s.Tests {
+			cmd := s.resolve(test.Command)
+			if !strings.Contains(cmd, "{{") {
+				nb++
+				if dryRun {
+					logging.Bullet(cmd)
+				} else {
+					cmdErr := s.run(cmd, strings.Join(test.Stdin, "\n"))
+					if cmdErr != nil {
+						// TODO: log the error
+						logging.Error("%v", cmdErr)
+					} else {
+						logging.Success(cmd)
+					}
+				}
+			}
+		}
+		return 1, nb, err
+	}
+	return 0, nb, err
+}
+
+func (s *Scenario) run(cmdline string, in string) error {
+	// Running the command in a shell ensure it does not run confined under the sudo profile.
+	// The shell is run unconfined and therefore the cmdline can be confined without no-new-privs issue.
+	sufix := " &" // TODO: we need a goroutine here
+	cmd := exec.Command("sh", "-c", cmdline+sufix)
+	if s.Root {
+		cmd = exec.Command("sudo", "sh", "-c", cmdline+sufix)
+	}
+	cmd.Stdin = strings.NewReader(in)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}

pkg/integration/suite.go

@@ -0,0 +1,79 @@
+// apparmor.d - Full set of apparmor profiles
+// Copyright (C) 2023 Alexandre Pujol <alexandre@pujol.io>
+// SPDX-License-Identifier: GPL-2.0-only
+
+package integration
+
+import (
+	"strings"
+
+	"github.com/arduino/go-paths-helper"
+	"github.com/roddhjav/apparmor.d/pkg/logs"
+	"gopkg.in/yaml.v2"
+)
+
+// TestSuite is the apparmod.d integration tests to run
+type TestSuite struct {
+	Scenarios []Scenario        // List of scenarios to run
+	Ignore    []string          // Do not run some scenarios
+	Arguments map[string]string // Common arguments used across all scenarios
+}
+
+// NewScenarios returns a new list of scenarios
+func NewTestSuite() *TestSuite {
+	return &TestSuite{
+		Scenarios: []Scenario{},
+		Ignore:    []string{},
+		Arguments: map[string]string{},
+	}
+}
+
+// Write export the list of scenarios to a file
+func (t *TestSuite) Write(path *paths.Path) error {
+	jsonString, err := yaml.Marshal(&t.Scenarios)
+	if err != nil {
+		return err
+	}
+
+	path = path.Clean()
+	file, err := path.Create()
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	// Cleanup a bit
+	res := string(jsonString)
+	res = strings.Replace(res, "- name:", "\n- name:", -1)
+	_, err = file.WriteString("---\n" + res)
+	return err
+}
+
+// ReadScenarios import the scenarios from a file
+func (t *TestSuite) ReadScenarios(path *paths.Path) error {
+	content, _ := path.ReadFile()
+	return yaml.Unmarshal(content, &t.Scenarios)
+}
+
+// ReadSettings import the common argument and ignore list from a file
+func (t *TestSuite) ReadSettings(path *paths.Path) error {
+	type temp struct {
+		Arguments map[string]string `yaml:"args"`
+		Ignore    []string          `yaml:"ignore"`
+	}
+	tmp := temp{}
+	content, _ := path.ReadFile()
+	if err := yaml.Unmarshal(content, &tmp); err != nil {
+		return err
+	}
+	t.Arguments = tmp.Arguments
+	t.Ignore = tmp.Ignore
+	return nil
+}
+
+// Results returns a sum up of the apparmor logs raised by the scenarios
+func (t *TestSuite) Results() string {
+	file, _ := logs.GetAuditLogs(logs.LogFiles[0])
+	aaLogs := logs.NewApparmorLogs(file, "")
+	return aaLogs.String()
+}

pkg/integration/tldr.go

@@ -0,0 +1,100 @@
+// apparmor.d - Full set of apparmor profiles
+// Copyright (C) 2023 Alexandre Pujol <alexandre@pujol.io>
+// SPDX-License-Identifier: GPL-2.0-only
+
+package integration
+
+import (
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/arduino/go-paths-helper"
+	"github.com/pkg/errors"
+	"github.com/roddhjav/apparmor.d/pkg/util"
+)
+
+var (
+	Ignore    []string          // Do not run some scenarios
+	Arguments map[string]string // Common arguments used across all scenarios
+)
+
+type Tldr struct {
+	Url    string      // Tldr download url
+	Dir    *paths.Path // Tldr cache directory
+	Ignore []string    // List of ignored software
+}
+
+func NewTldr(dir *paths.Path) Tldr {
+	return Tldr{
+		Url: "https://github.com/tldr-pages/tldr/archive/refs/heads/main.tar.gz",
+		Dir: dir,
+	}
+}
+
+// Download and extract the tldr pages into the cache directory
+func (t Tldr) Download() error {
+	gzPath := t.Dir.Parent().Join("tldr.tar.gz")
+	if !gzPath.Exist() {
+		resp, err := http.Get(t.Url)
+		if err != nil {
+			return errors.Wrapf(err, "downloading %s", t.Url)
+		}
+		defer resp.Body.Close()
+
+		out, err := gzPath.Create()
+		if err != nil {
+			return err
+		}
+		defer out.Close()
+
+		if _, err := io.Copy(out, resp.Body); err != nil {
+			return err
+		}
+	}
+
+	pages := []string{"tldr-main/pages/linux", "tldr-main/pages/common"}
+	if err := util.ExtratTo(gzPath, t.Dir, pages); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Parse the tldr pages and return a list of scenarios
+func (t Tldr) Parse(profiles paths.PathList) (*TestSuite, error) {
+	testSuite := NewTestSuite()
+	files, _ := t.Dir.ReadDirRecursiveFiltered(nil, paths.FilterOutDirectories())
+	for _, path := range files {
+		content, err := path.ReadFile()
+		if err != nil {
+			return nil, err
+		}
+		raw := string(content)
+		scenario := &Scenario{
+			Name:      strings.TrimSuffix(path.Base(), ".md"),
+			Profiled:  false,
+			Root:      false,
+			Arguments: map[string]string{},
+			Tests:     []Test{},
+		}
+		scenario.Profiled = scenario.hasProfile(profiles)
+		if strings.Contains(raw, "sudo") {
+			scenario.Root = true
+		}
+		rawTests := strings.Split(raw, "\n-")[1:]
+		for _, test := range rawTests {
+			res := strings.Split(test, "\n")
+			dsc := strings.ReplaceAll(strings.Trim(res[0], " "), ":", "")
+			cmd := strings.Trim(strings.Trim(res[2], "`"), " ")
+			if scenario.Root {
+				cmd = strings.ReplaceAll(cmd, "sudo ", "")
+			}
+			scenario.Tests = append(scenario.Tests, Test{
+				Description: dsc,
+				Command:     cmd,
+			})
+		}
+		testSuite.Scenarios = append(testSuite.Scenarios, *scenario)
+	}
+	return testSuite, nil
+}

pkg/logging/logging.go

@@ -21,7 +21,8 @@ const (
 // Logging messages prefix
 const (
 	bulletText  = bold + " ⋅ " + reset
-	errorText   = boldRed + " ✗ Error: " + reset
+	fatalText   = boldRed + " ✗ Error: " + reset
+	errorText   = boldRed + " ✗ " + reset
 	successText = boldGreen + " ✓ " + reset
 	warningText = boldYellow + " ‼ " + reset
 )
@@ -78,9 +79,14 @@ func Warning(msg string, a ...interface{}) int {
 	return Print(Warningf(msg, a...))
 }
 
+// Fatalf returns a formatted error message
+func Error(msg string, a ...interface{}) int {
+	return Print(fmt.Sprintf("%s%s\n", errorText, fmt.Sprintf(msg, a...)))
+}
+
 // Fatalf returns a formatted error message
 func Fatalf(msg string, a ...interface{}) string {
-	return fmt.Sprintf("%s%s\n", errorText, fmt.Sprintf(msg, a...))
+	return fmt.Sprintf("%s%s\n", fatalText, fmt.Sprintf(msg, a...))
 }
 
 // Fatal is equivalent to Print() followed by a call to os.Exit(1).

pkg/logging/logging_test.go

@@ -93,6 +93,15 @@ func TestWarning(t *testing.T) {
 	}
 }
 
+func TestError(t *testing.T) {
+	msg := "Error message"
+	wantN := 30
+	gotN := Error(msg)
+	if gotN != wantN {
+		t.Errorf("Error() = %v, want %v", gotN, wantN)
+	}
+}
+
 func TestFatalf(t *testing.T) {
 	msg := "Error message"
 	want := "\033[1;31m ✗ Error: \033[0mError message\n"

pkg/util/paths.go

@@ -0,0 +1,72 @@
+// apparmor.d - Full set of apparmor profiles
+// Copyright (C) 2023 Alexandre Pujol <alexandre@pujol.io>
+// SPDX-License-Identifier: GPL-2.0-only
+
+package util
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"io"
+	"path/filepath"
+	"strings"
+
+	"github.com/arduino/go-paths-helper"
+	"github.com/pkg/errors"
+)
+
+// Either or not to extract the file
+func toExtrat(name string, subfolders []string) bool {
+	for _, subfolder := range subfolders {
+		if strings.HasPrefix(name, subfolder) {
+			return true
+		}
+	}
+	return false
+}
+
+// Extract part of an archive to a destination directory
+func ExtratTo(src *paths.Path, dst *paths.Path, subfolders []string) error {
+	gzIn, err := src.Open()
+	if err != nil {
+		return errors.Wrapf(err, "opening %s", src)
+	}
+	defer gzIn.Close()
+
+	in, err := gzip.NewReader(gzIn)
+	if err != nil {
+		return errors.Wrapf(err, "decoding %s", src)
+	}
+	defer in.Close()
+
+	if err := dst.MkdirAll(); err != nil {
+		return errors.Wrapf(err, "creating %s", src)
+	}
+
+	tarIn := tar.NewReader(in)
+	for true {
+		header, err := tarIn.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+
+		if header.Typeflag == tar.TypeReg {
+			if !toExtrat(header.Name, subfolders) {
+				continue
+			}
+			path := dst.Join(filepath.Base(header.Name))
+			file, err := path.Create()
+			if err != nil {
+				return errors.Wrapf(err, "creating %s", file.Name())
+			}
+			if _, err := io.Copy(file, tarIn); err != nil {
+				return errors.Wrapf(err, "extracting %s", file.Name())
+			}
+			file.Close()
+		}
+	}
+	return nil
+}