From 298360fff1ddc6d3ef1cf3ad09a60d4b1e2258ae Mon Sep 17 00:00:00 2001 From: Alexandre Pujol Date: Sat, 6 May 2023 13:23:16 +0100 Subject: [PATCH] test(integration): initial version of integration tests manager --- cmd/aa-test/main.go | 179 ++++++++++++++++++++++++++++++++++++ pkg/integration/scenario.go | 128 ++++++++++++++++++++++++++ pkg/integration/suite.go | 79 ++++++++++++++++ pkg/integration/tldr.go | 100 ++++++++++++++++++++ pkg/logging/logging.go | 10 +- pkg/logging/logging_test.go | 9 ++ pkg/util/paths.go | 72 +++++++++++++++ 7 files changed, 575 insertions(+), 2 deletions(-) create mode 100644 cmd/aa-test/main.go create mode 100644 pkg/integration/scenario.go create mode 100644 pkg/integration/suite.go create mode 100644 pkg/integration/tldr.go create mode 100644 pkg/util/paths.go diff --git a/cmd/aa-test/main.go b/cmd/aa-test/main.go new file mode 100644 index 00000000..84a0798c --- /dev/null +++ b/cmd/aa-test/main.go @@ -0,0 +1,179 @@ +// apparmor.d - Full set of apparmor profiles +// Copyright (C) 2023 Alexandre Pujol +// SPDX-License-Identifier: GPL-2.0-only + +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/arduino/go-paths-helper" + intg "github.com/roddhjav/apparmor.d/pkg/integration" + "github.com/roddhjav/apparmor.d/pkg/logging" +) + +const usage = `aa-test [-h] [--bootstrap | --run | --list] + + Integration tests manager tool for apparmor.d + +Options: + -h, --help Show this help message and exit. + -b, --bootstrap Bootstrap test scenarios using tldr pages. + -r, --run Run a predefined list of tests. + -l, --list List the configured tests. + -f, --file FILE Set a scenario test file. Default: tests/scenarios.yml + +` + +const defaultScenariosFile = "scenarios.yml" + +var ( + help bool + bootstrap bool + run bool + list bool + path string + Cfg Config +) + +type Config struct { + TldrDir *paths.Path // Default: tests/tldr + ScenariosDir *paths.Path // Default: tests + ScenariosFile *paths.Path // Default: tests/tldr.yml + ProfilesDir *paths.Path // Default: /etc/apparmor.d + Profiles paths.PathList +} + +func NewConfig() Config { + cfg := Config{ + TldrDir: paths.New("tests/tldr"), + ScenariosDir: paths.New("tests/"), + ProfilesDir: paths.New("/etc/apparmor.d"), + Profiles: paths.PathList{}, + } + cfg.ScenariosFile = cfg.ScenariosDir.Join("tldr.yml") + return cfg +} + +func init() { + Cfg = NewConfig() + files, _ := Cfg.ProfilesDir.ReadDir(paths.FilterOutDirectories()) + for _, path := range files { + Cfg.Profiles.Add(path) + } + + flag.BoolVar(&help, "h", false, "Show this help message and exit.") + flag.BoolVar(&help, "help", false, "Show this help message and exit.") + flag.BoolVar(&bootstrap, "b", false, "Bootstrap test scenarios using tldr pages.") + flag.BoolVar(&bootstrap, "bootstrap", false, "Bootstrap test scenarios using tldr pages.") + flag.BoolVar(&run, "r", false, "Run a predefined list of tests.") + flag.BoolVar(&run, "run", false, "Run a predefined list of tests.") + flag.BoolVar(&list, "l", false, "List the test to run.") + flag.BoolVar(&list, "list", false, "List the test to run.") + flag.StringVar(&path, "f", defaultScenariosFile, "Set a scenario test file.") + flag.StringVar(&path, "file", defaultScenariosFile, "Set a scenario test file.") +} + +func apparmorTestBootstrap() error { + tldr := intg.NewTldr(Cfg.TldrDir) + if err := tldr.Download(); err != nil { + return err + } + + tSuite, err := tldr.Parse(Cfg.Profiles) + if err != nil { + return err + } + + // Default bootstraped scenarios file + if err := tSuite.Write(Cfg.ScenariosFile); err != nil { + return err + } + logging.Bullet("Default scenarios saved: %s", Cfg.ScenariosFile) + + // Scenarios file with only profiled programs + tSuiteWithProfile := intg.NewTestSuite() + for _, s := range tSuite.Scenarios { + if s.Profiled { + tSuiteWithProfile.Scenarios = append(tSuiteWithProfile.Scenarios, s) + } + } + scnPath := Cfg.ScenariosDir.Join(strings.TrimSuffix(path, filepath.Ext(path)) + ".new.yml") + if err := tSuiteWithProfile.Write(scnPath); err != nil { + return err + } + logging.Bullet("Scenarios with profile saved: %s", scnPath) + + logging.Bullet("Number of scenarios found %d", len(tSuite.Scenarios)) + logging.Bullet("Number of scenarios with profiles in apparmor.d %d", len(tSuiteWithProfile.Scenarios)) + return nil +} + +func apparmorTestRun(dryRun bool) error { + // FIXME: Safety settings. For default scenario set with sudo enabled the apparmor.d folder gets removed. + dryRun = true + if dryRun { + logging.Step("List tests") + } else { + logging.Step("Run tests") + } + + tSuite := intg.NewTestSuite() + scnPath := Cfg.ScenariosDir.Join(path) + if err := tSuite.ReadScenarios(scnPath); err != nil { + return err + } + cfgPath := Cfg.ScenariosDir.Join("integration.yml") + if err := tSuite.ReadSettings(cfgPath); err != nil { + return err + } + intg.Arguments = tSuite.Arguments + intg.Ignore = tSuite.Ignore + + nbTest := 0 + nbScn := 0 + for _, scn := range tSuite.Scenarios { + ran, nb, err := scn.Run(dryRun) + nbScn += ran + nbTest += nb + if err != nil { + return err + } + } + + if dryRun { + logging.Bullet("Number of scenarios to run %d", nbScn) + logging.Bullet("Number of tests to run %d", nbTest) + } else { + logging.Success("Number of scenarios ran %d", nbScn) + logging.Success("Number of tests to ran %d", nbTest) + } + return nil +} + +func main() { + flag.Usage = func() { fmt.Print(usage) } + flag.Parse() + if help { + flag.Usage() + os.Exit(0) + } + + var err error + if bootstrap { + logging.Step("Bootstraping tests") + err = apparmorTestBootstrap() + } else if run || list { + err = apparmorTestRun(list) + } else { + flag.Usage() + os.Exit(1) + } + if err != nil { + logging.Fatal(err.Error()) + } +} diff --git a/pkg/integration/scenario.go b/pkg/integration/scenario.go new file mode 100644 index 00000000..65a4b243 --- /dev/null +++ b/pkg/integration/scenario.go @@ -0,0 +1,128 @@ +// apparmor.d - Full set of apparmor profiles +// Copyright (C) 2023 Alexandre Pujol +// SPDX-License-Identifier: GPL-2.0-only + +// TODO: +// - Finish templating +// - Provide a large selection of resources: files, disks, http server... for automatic test on them +// - Expand support for interactive program (stdin and Control-D) +// - Properlly log the test result +// - Dbus integration + +package integration + +import ( + "os" + "os/exec" + "strings" + + "github.com/arduino/go-paths-helper" + "github.com/roddhjav/apparmor.d/pkg/logging" + "golang.org/x/exp/slices" +) + +// Scenario represents of a list of tests for a given program +type ( + Scenario struct { + Name string `yaml:"name"` + Profiled bool `yaml:"profiled"` // The program is profiled in apparmor.d + Root bool `yaml:"root"` // Run the test as user or as root + Dependencies []string `yaml:"require"` // Packages required for the tests to run "$(pacman -Qqo Scenario.Name)" + Arguments map[string]string `yaml:"arguments"` // Arguments to pass to the program. Sepicific to this scenario + Tests []Test `yaml:"tests"` + } + Test struct { + Description string `yaml:"dsc"` + Command string `yaml:"cmd"` + Stdin []string `yaml:"stdin"` + } +) + +func NewScenario() *Scenario { + return &Scenario{ + Name: "", + Profiled: false, + Root: false, + Dependencies: []string{}, + Arguments: map[string]string{}, + Tests: []Test{}, + } +} + +// HasProfile returns true if the program in the scenario is profiled in apparmor.d +func (s *Scenario) hasProfile(profiles paths.PathList) bool { + for _, path := range profiles { + if s.Name == path.Base() { + return true + } + } + return false +} + +func (s *Scenario) installed() bool { + if _, err := exec.LookPath(s.Name); err != nil { + return false + } + return true +} + +func (s *Scenario) resolve(in string) string { + res := in + for key, value := range s.Arguments { + res = strings.ReplaceAll(res, "{{"+key+"}}", value) + } + return res +} + +// mergeArguments merge the arguments of the scenario with the global arguments +// Scenarios arguments have priority over global arguments +func (s *Scenario) mergeArguments(args map[string]string) { + for key, value := range args { + s.Arguments[key] = value + } +} + +// Run the scenarios tests +func (s *Scenario) Run(dryRun bool) (ran int, nb int, err error) { + nb = 0 + if s.Profiled && s.installed() { + if slices.Contains(Ignore, s.Name) { + return 0, nb, err + } + logging.Step("%s", s.Name) + s.mergeArguments(Arguments) + for _, test := range s.Tests { + cmd := s.resolve(test.Command) + if !strings.Contains(cmd, "{{") { + nb++ + if dryRun { + logging.Bullet(cmd) + } else { + cmdErr := s.run(cmd, strings.Join(test.Stdin, "\n")) + if cmdErr != nil { + // TODO: log the error + logging.Error("%v", cmdErr) + } else { + logging.Success(cmd) + } + } + } + } + return 1, nb, err + } + return 0, nb, err +} + +func (s *Scenario) run(cmdline string, in string) error { + // Running the command in a shell ensure it does not run confined under the sudo profile. + // The shell is run unconfined and therefore the cmdline can be confined without no-new-privs issue. + sufix := " &" // TODO: we need a goroutine here + cmd := exec.Command("sh", "-c", cmdline+sufix) + if s.Root { + cmd = exec.Command("sudo", "sh", "-c", cmdline+sufix) + } + cmd.Stdin = strings.NewReader(in) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} diff --git a/pkg/integration/suite.go b/pkg/integration/suite.go new file mode 100644 index 00000000..f5e8321d --- /dev/null +++ b/pkg/integration/suite.go @@ -0,0 +1,79 @@ +// apparmor.d - Full set of apparmor profiles +// Copyright (C) 2023 Alexandre Pujol +// SPDX-License-Identifier: GPL-2.0-only + +package integration + +import ( + "strings" + + "github.com/arduino/go-paths-helper" + "github.com/roddhjav/apparmor.d/pkg/logs" + "gopkg.in/yaml.v2" +) + +// TestSuite is the apparmod.d integration tests to run +type TestSuite struct { + Scenarios []Scenario // List of scenarios to run + Ignore []string // Do not run some scenarios + Arguments map[string]string // Common arguments used across all scenarios +} + +// NewScenarios returns a new list of scenarios +func NewTestSuite() *TestSuite { + return &TestSuite{ + Scenarios: []Scenario{}, + Ignore: []string{}, + Arguments: map[string]string{}, + } +} + +// Write export the list of scenarios to a file +func (t *TestSuite) Write(path *paths.Path) error { + jsonString, err := yaml.Marshal(&t.Scenarios) + if err != nil { + return err + } + + path = path.Clean() + file, err := path.Create() + if err != nil { + return err + } + defer file.Close() + + // Cleanup a bit + res := string(jsonString) + res = strings.Replace(res, "- name:", "\n- name:", -1) + _, err = file.WriteString("---\n" + res) + return err +} + +// ReadScenarios import the scenarios from a file +func (t *TestSuite) ReadScenarios(path *paths.Path) error { + content, _ := path.ReadFile() + return yaml.Unmarshal(content, &t.Scenarios) +} + +// ReadSettings import the common argument and ignore list from a file +func (t *TestSuite) ReadSettings(path *paths.Path) error { + type temp struct { + Arguments map[string]string `yaml:"args"` + Ignore []string `yaml:"ignore"` + } + tmp := temp{} + content, _ := path.ReadFile() + if err := yaml.Unmarshal(content, &tmp); err != nil { + return err + } + t.Arguments = tmp.Arguments + t.Ignore = tmp.Ignore + return nil +} + +// Results returns a sum up of the apparmor logs raised by the scenarios +func (t *TestSuite) Results() string { + file, _ := logs.GetAuditLogs(logs.LogFiles[0]) + aaLogs := logs.NewApparmorLogs(file, "") + return aaLogs.String() +} diff --git a/pkg/integration/tldr.go b/pkg/integration/tldr.go new file mode 100644 index 00000000..91a462ac --- /dev/null +++ b/pkg/integration/tldr.go @@ -0,0 +1,100 @@ +// apparmor.d - Full set of apparmor profiles +// Copyright (C) 2023 Alexandre Pujol +// SPDX-License-Identifier: GPL-2.0-only + +package integration + +import ( + "io" + "net/http" + "strings" + + "github.com/arduino/go-paths-helper" + "github.com/pkg/errors" + "github.com/roddhjav/apparmor.d/pkg/util" +) + +var ( + Ignore []string // Do not run some scenarios + Arguments map[string]string // Common arguments used across all scenarios +) + +type Tldr struct { + Url string // Tldr download url + Dir *paths.Path // Tldr cache directory + Ignore []string // List of ignored software +} + +func NewTldr(dir *paths.Path) Tldr { + return Tldr{ + Url: "https://github.com/tldr-pages/tldr/archive/refs/heads/main.tar.gz", + Dir: dir, + } +} + +// Download and extract the tldr pages into the cache directory +func (t Tldr) Download() error { + gzPath := t.Dir.Parent().Join("tldr.tar.gz") + if !gzPath.Exist() { + resp, err := http.Get(t.Url) + if err != nil { + return errors.Wrapf(err, "downloading %s", t.Url) + } + defer resp.Body.Close() + + out, err := gzPath.Create() + if err != nil { + return err + } + defer out.Close() + + if _, err := io.Copy(out, resp.Body); err != nil { + return err + } + } + + pages := []string{"tldr-main/pages/linux", "tldr-main/pages/common"} + if err := util.ExtratTo(gzPath, t.Dir, pages); err != nil { + return err + } + return nil +} + +// Parse the tldr pages and return a list of scenarios +func (t Tldr) Parse(profiles paths.PathList) (*TestSuite, error) { + testSuite := NewTestSuite() + files, _ := t.Dir.ReadDirRecursiveFiltered(nil, paths.FilterOutDirectories()) + for _, path := range files { + content, err := path.ReadFile() + if err != nil { + return nil, err + } + raw := string(content) + scenario := &Scenario{ + Name: strings.TrimSuffix(path.Base(), ".md"), + Profiled: false, + Root: false, + Arguments: map[string]string{}, + Tests: []Test{}, + } + scenario.Profiled = scenario.hasProfile(profiles) + if strings.Contains(raw, "sudo") { + scenario.Root = true + } + rawTests := strings.Split(raw, "\n-")[1:] + for _, test := range rawTests { + res := strings.Split(test, "\n") + dsc := strings.ReplaceAll(strings.Trim(res[0], " "), ":", "") + cmd := strings.Trim(strings.Trim(res[2], "`"), " ") + if scenario.Root { + cmd = strings.ReplaceAll(cmd, "sudo ", "") + } + scenario.Tests = append(scenario.Tests, Test{ + Description: dsc, + Command: cmd, + }) + } + testSuite.Scenarios = append(testSuite.Scenarios, *scenario) + } + return testSuite, nil +} diff --git a/pkg/logging/logging.go b/pkg/logging/logging.go index b27fb70f..0b5493bd 100644 --- a/pkg/logging/logging.go +++ b/pkg/logging/logging.go @@ -21,7 +21,8 @@ const ( // Logging messages prefix const ( bulletText = bold + " ⋅ " + reset - errorText = boldRed + " ✗ Error: " + reset + fatalText = boldRed + " ✗ Error: " + reset + errorText = boldRed + " ✗ " + reset successText = boldGreen + " ✓ " + reset warningText = boldYellow + " ‼ " + reset ) @@ -78,9 +79,14 @@ func Warning(msg string, a ...interface{}) int { return Print(Warningf(msg, a...)) } +// Fatalf returns a formatted error message +func Error(msg string, a ...interface{}) int { + return Print(fmt.Sprintf("%s%s\n", errorText, fmt.Sprintf(msg, a...))) +} + // Fatalf returns a formatted error message func Fatalf(msg string, a ...interface{}) string { - return fmt.Sprintf("%s%s\n", errorText, fmt.Sprintf(msg, a...)) + return fmt.Sprintf("%s%s\n", fatalText, fmt.Sprintf(msg, a...)) } // Fatal is equivalent to Print() followed by a call to os.Exit(1). diff --git a/pkg/logging/logging_test.go b/pkg/logging/logging_test.go index b95ee041..898d30c7 100644 --- a/pkg/logging/logging_test.go +++ b/pkg/logging/logging_test.go @@ -93,6 +93,15 @@ func TestWarning(t *testing.T) { } } +func TestError(t *testing.T) { + msg := "Error message" + wantN := 30 + gotN := Error(msg) + if gotN != wantN { + t.Errorf("Error() = %v, want %v", gotN, wantN) + } +} + func TestFatalf(t *testing.T) { msg := "Error message" want := "\033[1;31m ✗ Error: \033[0mError message\n" diff --git a/pkg/util/paths.go b/pkg/util/paths.go new file mode 100644 index 00000000..400fb414 --- /dev/null +++ b/pkg/util/paths.go @@ -0,0 +1,72 @@ +// apparmor.d - Full set of apparmor profiles +// Copyright (C) 2023 Alexandre Pujol +// SPDX-License-Identifier: GPL-2.0-only + +package util + +import ( + "archive/tar" + "compress/gzip" + "io" + "path/filepath" + "strings" + + "github.com/arduino/go-paths-helper" + "github.com/pkg/errors" +) + +// Either or not to extract the file +func toExtrat(name string, subfolders []string) bool { + for _, subfolder := range subfolders { + if strings.HasPrefix(name, subfolder) { + return true + } + } + return false +} + +// Extract part of an archive to a destination directory +func ExtratTo(src *paths.Path, dst *paths.Path, subfolders []string) error { + gzIn, err := src.Open() + if err != nil { + return errors.Wrapf(err, "opening %s", src) + } + defer gzIn.Close() + + in, err := gzip.NewReader(gzIn) + if err != nil { + return errors.Wrapf(err, "decoding %s", src) + } + defer in.Close() + + if err := dst.MkdirAll(); err != nil { + return errors.Wrapf(err, "creating %s", src) + } + + tarIn := tar.NewReader(in) + for true { + header, err := tarIn.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + + if header.Typeflag == tar.TypeReg { + if !toExtrat(header.Name, subfolders) { + continue + } + path := dst.Join(filepath.Base(header.Name)) + file, err := path.Create() + if err != nil { + return errors.Wrapf(err, "creating %s", file.Name()) + } + if _, err := io.Copy(file, tarIn); err != nil { + return errors.Wrapf(err, "extracting %s", file.Name()) + } + file.Close() + } + } + return nil +}