Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

🌱 convert CITest check to probes #3621

Merged
merged 6 commits into from
Dec 11, 2023
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 11 additions & 7 deletions checks/ci_tests.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,10 @@
"github.com/ossf/scorecard/v4/checks/evaluation"
"github.com/ossf/scorecard/v4/checks/raw"
sce "github.com/ossf/scorecard/v4/errors"
"github.com/ossf/scorecard/v4/probes"
"github.com/ossf/scorecard/v4/probes/zrunner"
)

// CheckCodeReview is the registered name for DoesCodeReview.
const CheckCITests = "CI-Tests"

//nolint:gochecknoinits
Expand All @@ -35,19 +36,22 @@
}
}

// CodeReview will check if the maintainers perform code review.
func CITests(c *checker.CheckRequest) checker.CheckResult {
rawData, err := raw.CITests(c.RepoClient)
if err != nil {
e := sce.WithMessage(sce.ErrScorecardInternal, err.Error())
return checker.CreateRuntimeErrorResult(CheckCITests, e)
}

// Return raw results.
if c.RawResults != nil {
c.RawResults.CITestResults = rawData
pRawResults := getRawResults(c)
pRawResults.CITestResults = rawData

// Evaluate the probes.
findings, err := zrunner.Run(pRawResults, probes.CITests)
if err != nil {
e := sce.WithMessage(sce.ErrScorecardInternal, err.Error())
return checker.CreateRuntimeErrorResult(CheckCITests, e)

Check warning on line 53 in checks/ci_tests.go

View check run for this annotation

Codecov / codecov/patch

checks/ci_tests.go#L46-L53

Added lines #L46 - L53 were not covered by tests
}

// Return the score evaluation.
return evaluation.CITests(CheckCITests, &rawData, c.Dlogger)
return evaluation.CITests(CheckCITests, findings, c.Dlogger)

Check warning on line 56 in checks/ci_tests.go

View check run for this annotation

Codecov / codecov/patch

checks/ci_tests.go#L56

Added line #L56 was not covered by tests
}
137 changes: 45 additions & 92 deletions checks/evaluation/ci_tests.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,122 +16,75 @@

import (
"fmt"
"strings"

"github.com/ossf/scorecard/v4/checker"
sce "github.com/ossf/scorecard/v4/errors"
"github.com/ossf/scorecard/v4/finding"
"github.com/ossf/scorecard/v4/probes/testsRunInCI"
)

const (
// CheckCITests is the registered name for CITests.
CheckCITests = "CI-Tests"
success = "success"
)

func CITests(_ string, c *checker.CITestData, dl checker.DetailLogger) checker.CheckResult {
totalMerged := 0
totalTested := 0
for i := range c.CIInfo {
r := c.CIInfo[i]
totalMerged++

var foundCI bool

// GitHub Statuses.
prSuccessStatus, err := prHasSuccessStatus(r, dl)
if err != nil {
return checker.CreateRuntimeErrorResult(CheckCITests, err)
}
if prSuccessStatus {
totalTested++
foundCI = true
continue
}
const CheckCITests = "CI-Tests"

// GitHub Check Runs.
prCheckSuccessful, err := prHasSuccessfulCheck(r, dl)
if err != nil {
return checker.CreateRuntimeErrorResult(CheckCITests, err)
}
if prCheckSuccessful {
totalTested++
foundCI = true
}

if !foundCI {
// Log message says commit, but really we only care about PRs, and
// use only one commit (branch HEAD) to refer to all commits in a PR
func CITests(name string,
findings []finding.Finding,
dl checker.DetailLogger,
) checker.CheckResult {
expectedProbes := []string{
testsRunInCI.Probe,
}
if !finding.UniqueProbesEqual(findings, expectedProbes) {
e := sce.WithMessage(sce.ErrScorecardInternal, "invalid probe results")
return checker.CreateRuntimeErrorResult(name, e)
}

Check warning on line 38 in checks/evaluation/ci_tests.go

View check run for this annotation

Codecov / codecov/patch

checks/evaluation/ci_tests.go#L36-L38

Added lines #L36 - L38 were not covered by tests

// Debug PRs that were merged without CI tests
for i := range findings {
f := &findings[i]
if f.Outcome == finding.OutcomeNegative {
dl.Debug(&checker.LogMessage{
Text: fmt.Sprintf("merged PR %d without CI test at HEAD: %s", r.PullRequestNumber, r.HeadSHA),
Text: f.Message,
})
}
}

if totalMerged == 0 {
// check that the project has pull requests
if noPullRequestsFound(findings) {
return checker.CreateInconclusiveResult(CheckCITests, "no pull request found")
}

totalMerged, totalTested := getMergedAndTested(findings)

if totalMerged < totalTested || len(findings) < totalTested {
e := sce.WithMessage(sce.ErrScorecardInternal, "invalid finding values")
return checker.CreateRuntimeErrorResult(name, e)
}

Check warning on line 60 in checks/evaluation/ci_tests.go

View check run for this annotation

Codecov / codecov/patch

checks/evaluation/ci_tests.go#L58-L60

Added lines #L58 - L60 were not covered by tests

// Log all findings
checker.LogFindings(nonNegativeFindings(findings), dl)
AdamKorcz marked this conversation as resolved.
Show resolved Hide resolved

reason := fmt.Sprintf("%d out of %d merged PRs checked by a CI test", totalTested, totalMerged)
return checker.CreateProportionalScoreResult(CheckCITests, reason, totalTested, totalMerged)
}

// PR has a status marked 'success' and a CI-related context.
//
//nolint:unparam
func prHasSuccessStatus(r checker.RevisionCIInfo, dl checker.DetailLogger) (bool, error) {
for _, status := range r.Statuses {
if status.State != success {
continue
}
if isTest(status.Context) || isTest(status.TargetURL) {
dl.Debug(&checker.LogMessage{
Path: status.URL,
Type: finding.FileTypeURL,
Text: fmt.Sprintf("CI test found: pr: %s, context: %s", r.HeadSHA,
status.Context),
})
return true, nil
}
}
return false, nil
}
func getMergedAndTested(findings []finding.Finding) (int, int) {
totalMerged := 0
totalTested := 0

// PR has a successful CI-related check.
//
//nolint:unparam
func prHasSuccessfulCheck(r checker.RevisionCIInfo, dl checker.DetailLogger) (bool, error) {
for _, cr := range r.CheckRuns {
if cr.Status != "completed" {
continue
}
if cr.Conclusion != success {
continue
}
if isTest(cr.App.Slug) {
dl.Debug(&checker.LogMessage{
Path: cr.URL,
Type: finding.FileTypeURL,
Text: fmt.Sprintf("CI test found: pr: %d, context: %s", r.PullRequestNumber,
cr.App.Slug),
})
return true, nil
for i := range findings {
f := &findings[i]
totalMerged++
if f.Outcome == finding.OutcomePositive {
totalTested++
}
}
return false, nil
}

// isTest returns true if the given string is a CI test.
func isTest(s string) bool {
l := strings.ToLower(s)
return totalMerged, totalTested
}

// Add more patterns here!
for _, pattern := range []string{
"appveyor", "buildkite", "circleci", "e2e", "github-actions", "jenkins",
"mergeable", "packit-as-a-service", "semaphoreci", "test", "travis-ci",
"flutter-dashboard", "Cirrus CI", "azure-pipelines",
} {
if strings.Contains(l, pattern) {
func noPullRequestsFound(findings []finding.Finding) bool {
for i := range findings {
f := &findings[i]
if f.Outcome == finding.OutcomeNotApplicable {
return true
}
}
Expand Down
Loading
Loading