debug failing integration test

Signed-off-by: Adam Korczynski <[email protected]>

checks/evaluation/ci_tests.go

@@ -37,16 +37,26 @@ func CITests(name string,
 		return checker.CreateRuntimeErrorResult(name, e)
 	}
 
-	// check whether there are any negative findings
+	// Debug PRs that were merged without CI tests
+	for i := range findings {
+		f := &findings[i]
+		if f.Outcome == finding.OutcomeNegative {
+			dl.Debug(&checker.LogMessage{
+				Text: f.Message,
+			})
+		}
+	}
+
+	// check that the project has pull requests
 	if noPullRequestsFound(findings) {
 		return checker.CreateInconclusiveResult(CheckCITests, "no pull request found")
 	}
 
 	totalMerged := findings[0].Values["totalMerged"]
 	totalTested := findings[0].Values["totalTested"]
 
-	if totalMerged < totalTested || len(findings) != totalTested {
-		e := sce.WithMessage(sce.ErrScorecardInternal, "invalid probe results")
+	if totalMerged < totalTested || len(findings) < totalTested {
+		e := sce.WithMessage(sce.ErrScorecardInternal, "invalid finding values")
 		return checker.CreateRuntimeErrorResult(name, e)
 	}
 

checks/evaluation/ci_tests_test.go

@@ -124,6 +124,36 @@ func TestCITests(t *testing.T) {
 				Score:        -1,
 			},
 		},
+		{
+			name: "Tests debugging",
+			findings: []finding.Finding{
+				{
+					Outcome:  finding.OutcomeNegative,
+					Probe:    "testsRunInCI",
+					Message:  "merged PR 1 without CI test at HEAD: 1",
+					Values:   map[string]int{"totalMerged": 3, "totalTested": 0},
+					Location: &finding.Location{Type: 4},
+				},
+				{
+					Outcome:  finding.OutcomeNegative,
+					Probe:    "testsRunInCI",
+					Message:  "merged PR 1 without CI test at HEAD: 1",
+					Values:   map[string]int{"totalMerged": 3, "totalTested": 0},
+					Location: &finding.Location{Type: 4},
+				},
+				{
+					Outcome:  finding.OutcomeNegative,
+					Probe:    "testsRunInCI",
+					Message:  "merged PR 1 without CI test at HEAD: 1",
+					Values:   map[string]int{"totalMerged": 3, "totalTested": 0},
+					Location: &finding.Location{Type: 4},
+				},
+			},
+			result: scut.TestReturn{
+				NumberOfDebug: 3,
+				Score:         0,
+			},
+		},
 	}
 	for _, tt := range tests {
 		tt := tt

probes/testsRunInCI/impl.go

@@ -79,6 +79,16 @@ func Run(raw *checker.RawResults) ([]finding.Finding, string, error) {
 			findings = append(findings, *f)
 			totalTested++
 		}
+
+		if !prSuccessStatus && !prCheckSuccessful {
+			f, err := finding.NewWith(fs, Probe,
+				fmt.Sprintf("merged PR %d without CI test at HEAD: %s", r.PullRequestNumber, r.HeadSHA),
+				nil, finding.OutcomeNegative)
+			if err != nil {
+				return nil, Probe, fmt.Errorf("create finding: %w", err)
+			}
+			findings = append(findings, *f)
+		}
 	}
 
 	for i := range findings {