Skip to content

Commit

Permalink
🌱 speedup slowest e2e tests (ossf#3656)
Browse files Browse the repository at this point in the history 
* switch ossfuzz test to smaller repo

tensorflow/tensorflow is huge, and this causes the test to take forever.
locally this reduces the test time from 17 to 2.4 seconds

Signed-off-by: Spencer Schrock <[email protected]>

* reuse scorecard results for scorecard attestor policies

previously this test took 27 seconds locally, and now takes 8.
which is split across 3 subtests:
good repos: 1s
bad repos: 5s
code review policies: 2s

Signed-off-by: Spencer Schrock <[email protected]>

---------

Signed-off-by: Spencer Schrock <[email protected]>
  • Loading branch information
@spencerschrock
spencerschrock authored Nov 9, 2023
1 parent 0fc8296 commit 2c959b7
Show file tree
Hide file tree
Showing 2 changed files with 112 additions and 46 deletions.
154 changes: 110 additions & 44 deletions e2e/attestor_policy_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
package e2e

import (
"context"
"fmt"
"os"
"strings"

Expand All @@ -24,11 +26,16 @@ import (

"github.com/ossf/scorecard/v4/attestor/command"
"github.com/ossf/scorecard/v4/attestor/policy"
"github.com/ossf/scorecard/v4/checker"
"github.com/ossf/scorecard/v4/checks"
"github.com/ossf/scorecard/v4/clients"
sclog "github.com/ossf/scorecard/v4/log"
"github.com/ossf/scorecard/v4/pkg"
)

var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
Context("E2E TEST:Validating scorecard attestation policy", func() {
It("Should attest to repos based on policy", func() {
It("Should attest to known good repos based on policy", func() {
tt := []struct {
name string
repoURL string
Expand All @@ -46,60 +53,85 @@ var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
},
expected: policy.Pass,
},
}

for _, tc := range tt {
f, err := os.CreateTemp("/tmp", strings.ReplaceAll(tc.name, " ", "-"))
Expect(err).Should(BeNil())
defer os.Remove(f.Name())

buf, err := yaml.Marshal(tc.policy)
Expect(err).Should(BeNil())

nbytes, err := f.Write(buf)
Expect(err).Should(BeNil())
Expect(nbytes).Should(BeNumerically(">", 0))

result, err := command.RunCheckWithParams(tc.repoURL, tc.commit, f.Name())
Expect(err).Should(BeNil())
Expect(result).Should(BeEquivalentTo(tc.expected))
}
})
})
})

var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
Context("E2E TEST:Validating scorecard attestation policy", func() {
It("Should attest to bad repos based on policy", func() {
tt := []struct {
name string
repoURL string
commit string
policy policy.AttestationPolicy
expected policy.PolicyResult
}{
{
name: "test bad repo with vulnerabilities prevented but no known vulnerabilities",
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
name: "test bad repo with vulnerabilities prevented but no known vulnerabilities",
policy: policy.AttestationPolicy{
PreventKnownVulnerabilities: true,
},
expected: policy.Pass,
},
{
name: "test bad repo with ignored binary artifact",
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
name: "test bad repo with ignored binary artifact",
policy: policy.AttestationPolicy{
PreventBinaryArtifacts: true,
AllowedBinaryArtifacts: []string{"test-binary-artifact-*"},
},
expected: policy.Pass,
},
{
name: "test bad repo with binary artifact",
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
name: "test bad repo with binary artifact",
policy: policy.AttestationPolicy{
PreventBinaryArtifacts: true,
},
expected: policy.Fail,
},
{
name: "test bad repo with ignored dep by path",
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
name: "test bad repo with ignored dep by path",
policy: policy.AttestationPolicy{
PreventUnpinnedDependencies: true,
AllowedUnpinnedDependencies: []policy.Dependency{{Filepath: "Dockerfile"}},
},
expected: policy.Pass,
},
{
name: "test bad repo without ignored dep",
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
name: "test bad repo without ignored dep",
policy: policy.AttestationPolicy{
PreventUnpinnedDependencies: true,
},
expected: policy.Fail,
},
{
name: "test bad repo with ignored dep by name",
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
name: "test bad repo with ignored dep by name",
policy: policy.AttestationPolicy{
PreventUnpinnedDependencies: true,
AllowedUnpinnedDependencies: []policy.Dependency{{PackageName: "static-debian11"}, {PackageName: "golang"}},
},
expected: policy.Pass,
},
{
name: "test bad repo with everything ignored",
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
name: "test bad repo with everything ignored",
policy: policy.AttestationPolicy{
PreventBinaryArtifacts: true,
AllowedBinaryArtifacts: []string{"test-binary-artifact-*"},
Expand All @@ -110,27 +142,46 @@ var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
expected: policy.Pass,
},
{
name: "test repo with simple code review requirements",
repoURL: "https://github.com/ossf-tests/scorecard-attestor-code-review-e2e",
name: "test code reviews required but repo doesn't have code reviews",
policy: policy.AttestationPolicy{
EnsureCodeReviewed: true,
CodeReviewRequirements: policy.CodeReviewRequirements{
MinReviewers: 1,
},
},
expected: policy.Pass,
expected: policy.Fail,
},
}
results, err := getScorecardResult("https://github.com/ossf-tests/scorecard-binauthz-test-bad")
Expect(err).Should(BeNil())
for _, tc := range tt {
got, err := tc.policy.EvaluateResults(&results.RawResults)
Expect(err).Should(BeNil())
Expect(got).Should(BeEquivalentTo(tc.expected))
}
})
})
})

var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
Context("E2E TEST:Validating scorecard attestation policy", func() {
It("Should attest to repos based on code review policy", func() {
tt := []struct {
name string
repoURL string
commit string
policy policy.AttestationPolicy
expected policy.PolicyResult
}{
{
name: "test code reviews required but repo doesn't have code reviews",
repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
name: "test repo with simple code review requirements",
policy: policy.AttestationPolicy{
EnsureCodeReviewed: true,
CodeReviewRequirements: policy.CodeReviewRequirements{
MinReviewers: 1,
},
},
expected: policy.Fail,
expected: policy.Pass,
},
{
name: "test code reviews required with min reviewers",
repoURL: "https://github.com/ossf-tests/scorecard-attestor-code-review-e2e",
name: "test code reviews required with min reviewers",
policy: policy.AttestationPolicy{
EnsureCodeReviewed: true,
CodeReviewRequirements: policy.CodeReviewRequirements{
Expand All @@ -140,8 +191,7 @@ var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
expected: policy.Pass,
},
{
name: "test code reviews required with min reviewers and required reviewers",
repoURL: "https://github.com/ossf-tests/scorecard-attestor-code-review-e2e",
name: "test code reviews required with min reviewers and required reviewers",
policy: policy.AttestationPolicy{
EnsureCodeReviewed: true,
CodeReviewRequirements: policy.CodeReviewRequirements{
Expand All @@ -152,8 +202,7 @@ var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
expected: policy.Pass,
},
{
name: "test code reviews required with too many min reviewers but matching required reviewers",
repoURL: "https://github.com/ossf-tests/scorecard-attestor-code-review-e2e",
name: "test code reviews required with too many min reviewers but matching required reviewers",
policy: policy.AttestationPolicy{
EnsureCodeReviewed: true,
CodeReviewRequirements: policy.CodeReviewRequirements{
Expand All @@ -164,23 +213,40 @@ var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
expected: policy.Fail,
},
}

results, err := getScorecardResult("https://github.com/ossf-tests/scorecard-attestor-code-review-e2e")
Expect(err).Should(BeNil())
for _, tc := range tt {
f, err := os.CreateTemp("/tmp", strings.ReplaceAll(tc.name, " ", "-"))
Expect(err).Should(BeNil())
defer os.Remove(f.Name())

buf, err := yaml.Marshal(tc.policy)
Expect(err).Should(BeNil())

nbytes, err := f.Write(buf)
got, err := tc.policy.EvaluateResults(&results.RawResults)
Expect(err).Should(BeNil())
Expect(nbytes).Should(BeNumerically(">", 0))

result, err := command.RunCheckWithParams(tc.repoURL, tc.commit, f.Name())
Expect(err).Should(BeNil())
Expect(result).Should(BeEquivalentTo(tc.expected))
Expect(got).Should(BeEquivalentTo(tc.expected))
}
})
})
})

func getScorecardResult(repoURL string) (pkg.ScorecardResult, error) {
ctx := context.Background()
logger := sclog.NewLogger(sclog.DefaultLevel)

enabledChecks := map[string]checker.Check{
checks.CheckBinaryArtifacts: {
Fn: checks.BinaryArtifacts,
},
checks.CheckVulnerabilities: {
Fn: checks.Vulnerabilities,
},
checks.CheckCodeReview: {
Fn: checks.CodeReview,
},
checks.CheckPinnedDependencies: {
Fn: checks.PinningDependencies,
},
}
repo, repoClient, ossFuzzRepoClient, ciiClient, vulnsClient, err := checker.GetClients(
ctx, repoURL, "", logger)
if err != nil {
return pkg.ScorecardResult{}, fmt.Errorf("couldn't set up clients: %w", err)
}
//nolint:wrapcheck,lll
return pkg.RunScorecard(ctx, repo, clients.HeadSHA, 0, enabledChecks, repoClient, ossFuzzRepoClient, ciiClient, vulnsClient)
}
4 changes: 2 additions & 2 deletions e2e/fuzzing_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() {
Context("E2E TEST:Validating use of fuzzing tools", func() {
It("Should return use of OSS-Fuzz", func() {
dl := scut.TestDetailLogger{}
repo, err := githubrepo.MakeGithubRepo("tensorflow/tensorflow")
repo, err := githubrepo.MakeGithubRepo("ossf/scorecard-webapp")
Expect(err).Should(BeNil())
repoClient := githubrepo.CreateGithubRepoClient(context.Background(), logger)
err = repoClient.InitRepo(repo, clients.HeadSHA, 0)
Expand All @@ -53,7 +53,7 @@ var _ = Describe("E2E TEST:"+checks.CheckFuzzing, func() {
Error: nil,
Score: checker.MaxResultScore,
NumberOfWarn: 0,
NumberOfInfo: 12,
NumberOfInfo: 3, // 1 for OSSFuzz, 2 for go native fuzzing
NumberOfDebug: 0,
}
result := checks.Fuzzing(&req)
Expand Down

0 comments on commit 2c959b7

Please sign in to comment.