Skip to content

Commit

Permalink
validation: Use prove(1) as a TAP harness
Browse files Browse the repository at this point in the history
Capture stdout and stderr from create invocations, because we don't
want to pollute the TAP output with things like runc's:

  Incorrect Usage.
  ...

(which is for some reason printed to stdout) or:

  runc: "create" requires exactly 1 argument(s)

which is printed to stderr.  Instead, show the captured stderr as a
diagnostic, and hide the stdout completely.  Unless stderr is empty,
in which case show stdout in case it contains something useful.

Most of these tests are broken because we aren't collecting the
container exit code or post-start stdout.  But the tests haven't been
doing that since the create/start split in 15577bd (add runtime
struct; add create test, 2017-08-24, opencontainers#447) anyway [1].  This commit
just makes that more obvious.

The patsubst and wildcard Makefile syntax is documented in [2].  The
$(VALIDATION_TESTS) rule uses the static pattern rule syntax [3].

And use 'console' instead of 'sh' in the README, because these are
shell sessions, not scripts.  See [4,5].  I don't have a working runc
locally, so the mock results based on a dummy runtime script.

[1]: opencontainers#447 (comment)
[2]: https://www.gnu.org/software/make/manual/html_node/Wildcard-Function.html
[3]: https://www.gnu.org/software/make/manual/html_node/Static-Usage.html#Static-Usage
[4]: https://help.github.com/articles/creating-and-highlighting-code-blocks/#syntax-highlighting
[5]: https://github.com/github/linguist/blob/v5.3.3/lib/linguist/languages.yml#L4249-L4260

Signed-off-by: W. Trevor King <[email protected]>
  • Loading branch information
wking committed Nov 30, 2017
1 parent 1c2dca0 commit e11b77f
Show file tree
Hide file tree
Showing 24 changed files with 679 additions and 376 deletions.
17 changes: 13 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,9 @@ BUILDTAGS=
RUNTIME ?= runc
COMMIT=$(shell git rev-parse HEAD 2> /dev/null || true)
VERSION := ${shell cat ./VERSION}
VALIDATION_TESTS ?= $(patsubst %.go,%.t,$(wildcard validation/*.go))

all: tool runtimetest
all: tool runtimetest validation-executables

tool:
go build -tags "$(BUILDTAGS)" -ldflags "-X main.gitCommit=${COMMIT} -X main.version=${VERSION}" -o oci-runtime-tool ./cmd/oci-runtime-tool
Expand Down Expand Up @@ -35,10 +36,18 @@ uninstall:
rm -f $(PREFIX)/share/bash-completion/completions/oci-runtime-tool

clean:
rm -f oci-runtime-tool runtimetest *.1
rm -f oci-runtime-tool runtimetest *.1 $(VALIDATION_TESTS)

localvalidation: runtimetest
RUNTIME=$(RUNTIME) go test -tags "$(BUILDTAGS)" ${TESTFLAGS} -v github.com/opencontainers/runtime-tools/validation
localvalidation:
RUNTIME=$(RUNTIME) prove $(VALIDATION_TESTS)

.PHONY: validation-executables
validation-executables: $(VALIDATION_TESTS)

.PRECIOUS: $(VALIDATION_TESTS)
.PHONY: $(VALIDATION_TESTS)
$(VALIDATION_TESTS): %.t: %.go
go build -tags "$(BUILDTAGS)" ${TESTFLAGS} -o $@ $<

.PHONY: test .gofmt .govet .golint

Expand Down
133 changes: 92 additions & 41 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ To build from source code, runtime-tools requires Go 1.7.x or above.
[`oci-runtime-tool generate`][generate.1] generates [configuration JSON][config.json] for an [OCI bundle][bundle].
[OCI-compatible runtimes][runtime-spec] like [runC][] expect to read the configuration from `config.json`.

```sh
```console
$ oci-runtime-tool generate --output config.json
$ cat config.json
{
Expand All @@ -22,63 +22,114 @@ $ cat config.json
[`oci-runtime-tool validate`][validate.1] validates an OCI bundle.
The error message will be printed if the OCI bundle failed the validation procedure.

```sh
```console
$ oci-runtime-tool generate
$ oci-runtime-tool validate
INFO[0000] Bundle validation succeeded.
```

## Testing OCI runtimes

```sh
The runtime validation suite uses [`prove`][prove], which is packaged for most distributions (for example, it is in [Debian's `perl` package][debian-perl] and [Gentoo's `dev-lang/perl` package][gentoo-perl]).
If you cannot install `prove`, you can probably run the test suite with another [TAP consumer][tap-consumers], although you'll have to edit the [`Makefile`](Makefile) to replace `prove`.

```console
$ make runtimetest validation-executables
$ sudo make RUNTIME=runc localvalidation
RUNTIME=runc go test -tags "" -v github.com/opencontainers/runtime-tools/validation
=== RUN TestValidateBasic
TAP version 13
ok 1 - root filesystem
ok 2 - hostname
ok 3 - mounts
ok 4 - capabilities
ok 5 - default symlinks
ok 6 - default devices
ok 7 - linux devices
ok 8 - linux process
ok 9 - masked paths
ok 10 - oom score adj
ok 11 - read only paths
ok 12 - rlimits
ok 13 - sysctls
ok 14 - uid mappings
ok 15 - gid mappings
1..15
--- PASS: TestValidateBasic (0.08s)
=== RUN TestValidateSysctls
RUNTIME=runc prove validation/linux_rootfs_propagation_shared.t validation/create.t validation/default.t validation/linux_readonly_paths.t validation/linux_masked_paths.t validation/mounts.t validation/process.t validation/root_readonly_false.t validation/linux_sysctl.t validation/linux_devices.t validation/linux_gid_mappings.t validation/process_oom_score_adj.t validation/process_capabilities.t validation/process_rlimits.t validation/root_readonly_true.t validation/linux_rootfs_propagation_unbindable.t validation/hostname.t validation/linux_uid_mappings.t
validation/linux_rootfs_propagation_shared.t ...... Failed 1/19 subtests
validation/create.t ............................... ok
validation/default.t .............................. ok
validation/linux_readonly_paths.t ................. ok
validation/linux_masked_paths.t ................... Failed 1/19 subtests
validation/mounts.t ............................... ok
validation/process.t .............................. ok
validation/root_readonly_false.t .................. ok
validation/linux_sysctl.t ......................... ok
validation/linux_devices.t ........................ ok
validation/linux_gid_mappings.t ................... Failed 1/19 subtests
validation/process_oom_score_adj.t ................ ok
validation/process_capabilities.t ................. ok
validation/process_rlimits.t ...................... ok
validation/root_readonly_true.t ................... ok
validation/linux_rootfs_propagation_unbindable.t .. failed to create the container
rootfsPropagation=unbindable is not supported
exit status 1
validation/linux_rootfs_propagation_unbindable.t .. Dubious, test returned 1 (wstat 256, 0x100)
No subtests run
validation/hostname.t ............................. ok
validation/linux_uid_mappings.t ................... failed to create the container
User namespace mappings specified, but USER namespace isn't enabled in the config
exit status 1
validation/linux_uid_mappings.t ................... Dubious, test returned 1 (wstat 256, 0x100)
No subtests run

Test Summary Report
-------------------
validation/linux_rootfs_propagation_shared.t (Wstat: 0 Tests: 19 Failed: 1)
Failed test: 16
validation/linux_masked_paths.t (Wstat: 0 Tests: 19 Failed: 1)
Failed test: 13
validation/linux_gid_mappings.t (Wstat: 0 Tests: 19 Failed: 1)
Failed test: 19
validation/linux_rootfs_propagation_unbindable.t (Wstat: 256 Tests: 0 Failed: 0)
Non-zero exit status: 1
Parse errors: No plan found in TAP output
validation/linux_uid_mappings.t (Wstat: 256 Tests: 0 Failed: 0)
Non-zero exit status: 1
Parse errors: No plan found in TAP output
Files=18, Tests=271, 31 wallclock secs ( 0.06 usr 0.01 sys + 0.52 cusr 0.21 csys = 0.80 CPU)
Result: FAIL
make: *** [Makefile:42: localvalidation] Error 1
```

If you are confident that the `validation/*.t` are current, you can run `prove` (or your preferred TAP consumer) directly to avoid unnecessary rebuilds:

```console
$ sudo RUNTIME=runc prove -Q validation/*.t

Test Summary Report
-------------------
Files=18, Tests=271, 31 wallclock secs ( 0.07 usr 0.01 sys + 0.51 cusr 0.22 csys = 0.81 CPU)
Result: FAIL
```

And you can run an individual test executable directly:

```console
$ RUNTIME=runc validation/default.t
TAP version 13
ok 1 - root filesystem
ok 2 - hostname
ok 3 - mounts
ok 4 - capabilities
ok 5 - default symlinks
ok 6 - default devices
ok 7 - linux devices
ok 8 - linux process
ok 9 - masked paths
ok 10 - oom score adj
ok 11 - read only paths
ok 12 - rlimits
ok 13 - sysctls
ok 14 - uid mappings
ok 15 - gid mappings
1..15
--- PASS: TestValidateSysctls (0.20s)
PASS
ok github.com/opencontainers/runtime-tools/validation 0.281s
ok 3 - process
ok 4 - mounts
ok 5 - user
ok 6 - rlimits
ok 7 - capabilities
ok 8 - default symlinks
ok 9 - default file system
ok 10 - default devices
ok 11 - linux devices
ok 12 - linux process
ok 13 - masked paths
ok 14 - oom score adj
ok 15 - read only paths
ok 16 - rootfs propagation
ok 17 - sysctls
ok 18 - uid mappings
ok 19 - gid mappings
1..19
```

[bundle]: https://github.com/opencontainers/runtime-spec/blob/master/bundle.md
[config.json]: https://github.com/opencontainers/runtime-spec/blob/master/config.md
[debian-perl]: https://packages.debian.org/stretch/perl
[gentoo-perl]: https://packages.gentoo.org/packages/dev-lang/perl
[prove]: http://search.cpan.org/~leont/Test-Harness-3.39/bin/prove
[runC]: https://github.com/opencontainers/runc
[runtime-spec]: https://github.com/opencontainers/runtime-spec
[tap-consumers]: https://testanything.org/consumers.html

[generate.1]: man/oci-runtime-tool-generate.1.md
[validate.1]: man/oci-runtime-tool-validate.1.md
1 change: 1 addition & 0 deletions validation/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
/*.t
68 changes: 68 additions & 0 deletions validation/create.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
package main

import (
"fmt"

"github.com/mndrix/tap-go"
rspecs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/opencontainers/runtime-tools/generate"
"github.com/opencontainers/runtime-tools/specerror"
"github.com/opencontainers/runtime-tools/validation/util"
"github.com/satori/go.uuid"
)

func main() {
t := tap.New()

g := generate.New()
g.SetRootPath(".")
g.SetProcessArgs([]string{"ls"})

bundleDir, err := util.PrepareBundle()
if err != nil {
util.Fatal(err)
}

r, err := util.NewRuntime(util.RuntimeCommand, bundleDir)
if err != nil {
util.Fatal(err)
}
defer r.Clean(true)

err = r.SetConfig(&g)
if err != nil {
util.Fatal(err)
}

containerID := uuid.NewV4().String()
cases := []struct {
id string
errExpected bool
err error
}{
{"", false, specerror.NewError(specerror.CreateWithBundlePathAndID, fmt.Errorf("create MUST generate an error if the ID is not provided"), rspecs.Version)},
{containerID, true, specerror.NewError(specerror.CreateNewContainer, fmt.Errorf("create MUST create a new container"), rspecs.Version)},
{containerID, false, specerror.NewError(specerror.CreateWithUniqueID, fmt.Errorf("create MUST generate an error if the ID provided is not unique"), rspecs.Version)},
}

for _, c := range cases {
r.SetID(c.id)
stderr, err := r.Create()
t.Ok((err == nil) == c.errExpected, c.err.(*specerror.Error).Err.Err.Error())
t.Diagnostic(c.err.(*specerror.Error).Err.Reference)
if err != nil {
t.Diagnostic(err.Error())
}
if len(stderr) > 0 {
t.Diagnostic(string(stderr))
}

if err == nil {
state, _ := r.State()
t.Ok(state.ID == c.id, "")
t.Diagnosticf("container PID: %d, state ID: %d", c.id, state.ID)
}
}

t.AutoPlan()
}
13 changes: 13 additions & 0 deletions validation/default.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package main

import (
"github.com/opencontainers/runtime-tools/validation/util"
)

func main() {
g := util.GetDefaultGenerator()
err := util.RuntimeInsideValidate(g, nil)
if err != nil {
util.Fatal(err)
}
}
14 changes: 14 additions & 0 deletions validation/hostname.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package main

import (
"github.com/opencontainers/runtime-tools/validation/util"
)

func main() {
g := util.GetDefaultGenerator()
g.SetHostname("hostname-specific")
err := util.RuntimeInsideValidate(g, nil)
if err != nil {
util.Fatal(err)
}
}
55 changes: 55 additions & 0 deletions validation/linux_devices.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
package main

import (
"os"

rspecs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/opencontainers/runtime-tools/validation/util"
)

func main() {
g := util.GetDefaultGenerator()

// add char device
cdev := rspecs.LinuxDevice{}
cdev.Path = "/dev/test1"
cdev.Type = "c"
cdev.Major = 10
cdev.Minor = 666
cmode := os.FileMode(int32(432))
cdev.FileMode = &cmode
cuid := uint32(0)
cdev.UID = &cuid
cgid := uint32(0)
cdev.GID = &cgid
g.AddDevice(cdev)

// add block device
bdev := rspecs.LinuxDevice{}
bdev.Path = "/dev/test2"
bdev.Type = "b"
bdev.Major = 8
bdev.Minor = 666
bmode := os.FileMode(int32(432))
bdev.FileMode = &bmode
uid := uint32(0)
bdev.UID = &uid
gid := uint32(0)
bdev.GID = &gid
g.AddDevice(bdev)

// add fifo device
pdev := rspecs.LinuxDevice{}
pdev.Path = "/dev/test3"
pdev.Type = "p"
pdev.Major = 8
pdev.Minor = 666
pmode := os.FileMode(int32(432))
pdev.FileMode = &pmode
g.AddDevice(pdev)

err := util.RuntimeInsideValidate(g, nil)
if err != nil {
util.Fatal(err)
}
}
14 changes: 14 additions & 0 deletions validation/linux_gid_mappings.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package main

import (
"github.com/opencontainers/runtime-tools/validation/util"
)

func main() {
g := util.GetDefaultGenerator()
g.AddLinuxGIDMapping(uint32(1000), uint32(0), uint32(3200))
err := util.RuntimeInsideValidate(g, nil)
if err != nil {
util.Fatal(err)
}
}
20 changes: 20 additions & 0 deletions validation/linux_masked_paths.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package main

import (
"os"
"path/filepath"

"github.com/opencontainers/runtime-tools/validation/util"
)

func main() {
g := util.GetDefaultGenerator()
g.AddLinuxMaskedPaths("/masktest")
err := util.RuntimeInsideValidate(g, func(path string) error {
pathName := filepath.Join(path, "masktest")
return os.MkdirAll(pathName, 0700)
})
if err != nil {
util.Fatal(err)
}
}
Loading

0 comments on commit e11b77f

Please sign in to comment.