🌱 Convert SAST check to probes (#3571)

* Convert SAST checks to probes Signed-off-by: AdamKorcz <[email protected]> * Update checks/evaluation/sast.go Co-authored-by: Raghav Kaul <[email protected]> Signed-off-by: AdamKorcz <[email protected]> * preserve file info when logging positive Sonar findings Signed-off-by: AdamKorcz <[email protected]> * rebase Signed-off-by: AdamKorcz <[email protected]> * Remove warning logging Signed-off-by: AdamKorcz <[email protected]> * add outcome and message to finding on the same line Signed-off-by: AdamKorcz <[email protected]> * codeql workflow -> codeql action Signed-off-by: AdamKorcz <[email protected]> * 'the Sonar' -> 'Sonar' in probe def.yml Signed-off-by: AdamKorcz <[email protected]> * fix typo Signed-off-by: AdamKorcz <[email protected]> * Change how probe creates location Signed-off-by: AdamKorcz <[email protected]> * Change names of values Signed-off-by: AdamKorcz <[email protected]> * change 'SAST tool detected: xx' to 'SAST tool installed: xx' Signed-off-by: AdamKorcz <[email protected]> * make text in probe def.yml easier to read Signed-off-by: AdamKorcz <[email protected]> * Change 'to' to 'two' Signed-off-by: AdamKorcz <[email protected]> * Minor change Signed-off-by: AdamKorcz <[email protected]> --------- Signed-off-by: AdamKorcz <[email protected]> Signed-off-by: AdamKorcz <[email protected]> Co-authored-by: Raghav Kaul <[email protected]>
ossf · Nov 7, 2023 · 47e04c1 · 47e04c1
1 parent f422f69
commit 47e04c1
Show file tree

Hide file tree

Showing 22 changed files with 1,729 additions and 542 deletions.
diff --git a/checker/raw_result.go b/checker/raw_result.go
@@ -39,6 +39,7 @@ type RawResults struct {
 	Metadata                    MetadataData
 	PackagingResults            PackagingData
 	PinningDependenciesResults  PinningDependenciesData
+	SASTResults                 SASTData
 	SecurityPolicyResults       SecurityPolicyData
 	SignedReleasesResults       SignedReleasesData
 	TokenPermissionsResults     TokenPermissionsData
@@ -226,6 +227,40 @@ type SecurityPolicyFile struct {
 	File File
 }
 
+// SASTData contains the raw results
+// for the SAST check.
+type SASTData struct {
+	Workflows    []SASTWorkflow
+	Commits      []SASTCommit
+	NumWorkflows int
+}
+
+type SASTCommit struct {
+	CommittedDate          time.Time
+	Message                string
+	SHA                    string
+	CheckRuns              []clients.CheckRun
+	AssociatedMergeRequest clients.PullRequest
+	Committer              clients.User
+	Compliant              bool
+}
+
+// SASTWorkflowType represents a type of SAST workflow.
+type SASTWorkflowType string
+
+const (
+	// CodeQLWorkflow represents a workflow that runs CodeQL.
+	CodeQLWorkflow SASTWorkflowType = "CodeQL"
+	// SonarWorkflow represents a workflow that runs Sonar.
+	SonarWorkflow SASTWorkflowType = "Sonar"
+)
+
+// SASTWorkflow represents a SAST workflow.
+type SASTWorkflow struct {
+	Type SASTWorkflowType
+	File File
+}
+
 // SecurityPolicyData contains the raw results
 // for the Security-Policy check.
 type SecurityPolicyData struct {

diff --git a/checks/evaluation/sast.go b/checks/evaluation/sast.go
@@ -0,0 +1,167 @@
+// Copyright 2023 OpenSSF Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package evaluation
+
+import (
+	"github.com/ossf/scorecard/v4/checker"
+	sce "github.com/ossf/scorecard/v4/errors"
+	"github.com/ossf/scorecard/v4/finding"
+	"github.com/ossf/scorecard/v4/probes/sastToolCodeQLInstalled"
+	"github.com/ossf/scorecard/v4/probes/sastToolRunsOnAllCommits"
+	"github.com/ossf/scorecard/v4/probes/sastToolSonarInstalled"
+)
+
+// SAST applies the score policy for the SAST check.
+func SAST(name string,
+	findings []finding.Finding, dl checker.DetailLogger,
+) checker.CheckResult {
+	// We have 3 unique probes, each should have a finding.
+	expectedProbes := []string{
+		sastToolCodeQLInstalled.Probe,
+		sastToolRunsOnAllCommits.Probe,
+		sastToolSonarInstalled.Probe,
+	}
+
+	if !finding.UniqueProbesEqual(findings, expectedProbes) {
+		e := sce.WithMessage(sce.ErrScorecardInternal, "invalid probe results")
+		return checker.CreateRuntimeErrorResult(name, e)
+	}
+
+	var sastScore, codeQlScore, sonarScore int
+	// Assign sastScore, codeQlScore and sonarScore
+	for i := range findings {
+		f := &findings[i]
+		switch f.Probe {
+		case sastToolRunsOnAllCommits.Probe:
+			sastScore = getSASTScore(f, dl)
+		case sastToolCodeQLInstalled.Probe:
+			codeQlScore = getCodeQLScore(f, dl)
+		case sastToolSonarInstalled.Probe:
+			if f.Outcome == finding.OutcomePositive {
+				sonarScore = checker.MaxResultScore
+				dl.Info(&checker.LogMessage{
+					Text:      f.Message,
+					Type:      f.Location.Type,
+					Path:      f.Location.Path,
+					Offset:    *f.Location.LineStart,
+					EndOffset: *f.Location.LineEnd,
+					Snippet:   *f.Location.Snippet,
+				})
+			} else if f.Outcome == finding.OutcomeNegative {
+				sonarScore = checker.MinResultScore
+			}
+		}
+	}
+
+	if sonarScore == checker.MaxResultScore {
+		return checker.CreateMaxScoreResult(name, "SAST tool detected")
+	}
+
+	if sastScore == checker.InconclusiveResultScore &&
+		codeQlScore == checker.InconclusiveResultScore {
+		// That can never happen since sastToolInCheckRuns can never
+		// retun checker.InconclusiveResultScore.
+		return checker.CreateRuntimeErrorResult(name, sce.ErrScorecardInternal)
+	}
+
+	// Both scores are conclusive.
+	// We assume the CodeQl config uses a cron and is not enabled as pre-submit.
+	// TODO: verify the above comment in code.
+	// We encourage developers to have sast check run on every pre-submit rather
+	// than as cron jobs through the score computation below.
+	// Warning: there is a hidden assumption that *any* sast tool is equally good.
+	if sastScore != checker.InconclusiveResultScore &&
+		codeQlScore != checker.InconclusiveResultScore {
+		switch {
+		case sastScore == checker.MaxResultScore:
+			return checker.CreateMaxScoreResult(name, "SAST tool is run on all commits")
+		case codeQlScore == checker.MinResultScore:
+			return checker.CreateResultWithScore(name,
+				checker.NormalizeReason("SAST tool is not run on all commits", sastScore), sastScore)
+
+		// codeQl is enabled and sast has 0+ (but not all) PRs checks.
+		case codeQlScore == checker.MaxResultScore:
+			const sastWeight = 3
+			const codeQlWeight = 7
+			score := checker.AggregateScoresWithWeight(map[int]int{sastScore: sastWeight, codeQlScore: codeQlWeight})
+			return checker.CreateResultWithScore(name, "SAST tool detected but not run on all commits", score)
+		default:
+			return checker.CreateRuntimeErrorResult(name, sce.WithMessage(sce.ErrScorecardInternal, "contact team"))
+		}
+	}
+
+	// Sast inconclusive.
+	if codeQlScore != checker.InconclusiveResultScore {
+		if codeQlScore == checker.MaxResultScore {
+			return checker.CreateMaxScoreResult(name, "SAST tool detected: CodeQL")
+		}
+		return checker.CreateMinScoreResult(name, "no SAST tool detected")
+	}
+
+	// CodeQl inconclusive.
+	if sastScore != checker.InconclusiveResultScore {
+		if sastScore == checker.MaxResultScore {
+			return checker.CreateMaxScoreResult(name, "SAST tool is run on all commits")
+		}
+
+		return checker.CreateResultWithScore(name,
+			checker.NormalizeReason("SAST tool is not run on all commits", sastScore), sastScore)
+	}
+
+	// Should never happen.
+	return checker.CreateRuntimeErrorResult(name, sce.WithMessage(sce.ErrScorecardInternal, "contact team"))
+}
+
+// getSASTScore returns the proportional score of how many commits
+// run SAST tools.
+func getSASTScore(f *finding.Finding, dl checker.DetailLogger) int {
+	switch f.Outcome {
+	case finding.OutcomeNotApplicable:
+		dl.Warn(&checker.LogMessage{
+			Text: f.Message,
+		})
+		return checker.InconclusiveResultScore
+	case finding.OutcomePositive:
+		dl.Info(&checker.LogMessage{
+			Text: f.Message,
+		})
+	case finding.OutcomeNegative:
+		dl.Warn(&checker.LogMessage{
+			Text: f.Message,
+		})
+	default:
+		checker.CreateProportionalScore(f.Values["totalPullRequestsAnalyzed"], f.Values["totalPullRequestsMerged"])
+	}
+	return checker.CreateProportionalScore(f.Values["totalPullRequestsAnalyzed"], f.Values["totalPullRequestsMerged"])
+}
+
+// getCodeQLScore returns positive the project runs CodeQL and negative
+// if it doesn't.
+func getCodeQLScore(f *finding.Finding, dl checker.DetailLogger) int {
+	switch f.Outcome {
+	case finding.OutcomePositive:
+		dl.Info(&checker.LogMessage{
+			Text: f.Message,
+		})
+		return checker.MaxResultScore
+	case finding.OutcomeNegative:
+		dl.Warn(&checker.LogMessage{
+			Text: f.Message,
+		})
+		return checker.MinResultScore
+	default:
+		panic("Should not happen")
+	}
+}
diff --git a/checks/evaluation/sast_test.go b/checks/evaluation/sast_test.go
@@ -0,0 +1,153 @@
+// Copyright 2023 OpenSSF Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package evaluation
+
+import (
+	"testing"
+
+	"github.com/ossf/scorecard/v4/checker"
+	sce "github.com/ossf/scorecard/v4/errors"
+	"github.com/ossf/scorecard/v4/finding"
+	scut "github.com/ossf/scorecard/v4/utests"
+)
+
+func TestSAST(t *testing.T) {
+	snippet := "some code snippet"
+	sline := uint(10)
+	eline := uint(46)
+	t.Parallel()
+	tests := []struct {
+		name     string
+		findings []finding.Finding
+		result   scut.TestReturn
+	}{
+		{
+			name: "SAST - Missing a probe",
+			findings: []finding.Finding{
+				{
+					Probe:   "sastToolCodeQLInstalled",
+					Outcome: finding.OutcomePositive,
+				},
+				{
+					Probe:   "sastToolRunsOnAllCommits",
+					Outcome: finding.OutcomePositive,
+				},
+			},
+			result: scut.TestReturn{
+				Score: checker.InconclusiveResultScore,
+				Error: sce.ErrScorecardInternal,
+			},
+		},
+		{
+			name: "Sonar and codeQL is installed",
+			findings: []finding.Finding{
+				{
+					Probe:   "sastToolCodeQLInstalled",
+					Outcome: finding.OutcomePositive,
+				},
+				{
+					Probe:   "sastToolRunsOnAllCommits",
+					Outcome: finding.OutcomePositive,
+					Values: map[string]int{
+						"totalPullRequestsAnalyzed": 1,
+						"totalPullRequestsMerged":   2,
+					},
+				},
+				{
+					Probe:   "sastToolSonarInstalled",
+					Outcome: finding.OutcomePositive,
+					Location: &finding.Location{
+						Type:      finding.FileTypeSource,
+						Path:      "path/to/file.txt",
+						LineStart: &sline,
+						LineEnd:   &eline,
+						Snippet:   &snippet,
+					},
+				},
+			},
+			result: scut.TestReturn{
+				Score:        10,
+				NumberOfInfo: 3,
+			},
+		},
+		{
+			name: `Sonar is installed. CodeQL is not installed.
+					Does not have info about whether SAST runs
+					on every commit.`,
+			findings: []finding.Finding{
+				{
+					Probe:   "sastToolCodeQLInstalled",
+					Outcome: finding.OutcomeNegative,
+				},
+				{
+					Probe:   "sastToolRunsOnAllCommits",
+					Outcome: finding.OutcomeNotApplicable,
+				},
+				{
+					Probe:   "sastToolSonarInstalled",
+					Outcome: finding.OutcomePositive,
+					Location: &finding.Location{
+						Type:      finding.FileTypeSource,
+						Path:      "path/to/file.txt",
+						LineStart: &sline,
+						LineEnd:   &eline,
+						Snippet:   &snippet,
+					},
+				},
+			},
+			result: scut.TestReturn{
+				Score:        10,
+				NumberOfInfo: 1,
+				NumberOfWarn: 2,
+			},
+		},
+		{
+			name: "Sonar and CodeQL are not installed",
+			findings: []finding.Finding{
+				{
+					Probe:   "sastToolCodeQLInstalled",
+					Outcome: finding.OutcomeNegative,
+				},
+				{
+					Probe:   "sastToolRunsOnAllCommits",
+					Outcome: finding.OutcomeNegative,
+					Values: map[string]int{
+						"totalPullRequestsAnalyzed": 1,
+						"totalPullRequestsMerged":   3,
+					},
+				},
+				{
+					Probe:   "sastToolSonarInstalled",
+					Outcome: finding.OutcomeNegative,
+				},
+			},
+			result: scut.TestReturn{
+				Score:        3,
+				NumberOfWarn: 2,
+				NumberOfInfo: 0,
+			},
+		},
+	}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			dl := scut.TestDetailLogger{}
+			got := SAST(tt.name, tt.findings, &dl)
+			if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) {
+				t.Errorf("got %v, expected %v", got, tt.result)
+			}
+		})
+	}
+}