Merge branch 'main' into fix/build_spectral_on_arm

goharbor · Jun 29, 2024 · 81db8f1 · 81db8f1
2 parents 1ace7ed + 562c01e
commit 81db8f1
Show file tree

Hide file tree

Showing 56 changed files with 443 additions and 323 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -5,7 +5,7 @@ env:
    POSTGRESQL_USR: postgres
    POSTGRESQL_PWD: root123
    POSTGRESQL_DATABASE: registry
-   DOCKER_COMPOSE_VERSION: 1.23.0
+   DOCKER_COMPOSE_VERSION: 2.27.1
    HARBOR_ADMIN: admin
    HARBOR_ADMIN_PASSWD: Harbor12345
    CORE_SECRET: tempString
@@ -66,7 +66,7 @@ jobs:
           env
           #sudo apt install -y xvfb
           #xvfb-run ls
-          curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
+          curl -L https://github.com/docker/compose/releases/download/v${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
           chmod +x docker-compose
           sudo mv docker-compose /usr/local/bin
           IP=`hostname -I | awk '{print $1}'`
@@ -131,7 +131,7 @@ jobs:
           df -h
           #sudo apt install -y xvfb
           #xvfb-run ls
-          curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
+          curl -L https://github.com/docker/compose/releases/download/v${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
           chmod +x docker-compose
           sudo mv docker-compose /usr/local/bin
       - name: install
@@ -186,7 +186,7 @@ jobs:
           df -h
           #sudo apt install -y xvfb
           #xvfb-run ls
-          curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
+          curl -L https://github.com/docker/compose/releases/download/v${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
           chmod +x docker-compose
           sudo mv docker-compose /usr/local/bin
       - name: install
@@ -240,7 +240,7 @@ jobs:
           df -h
           #sudo apt install -y xvfb
           #xvfb-run ls
-          curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
+          curl -L https://github.com/docker/compose/releases/download/v${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
           chmod +x docker-compose
           sudo mv docker-compose /usr/local/bin
       - name: install
@@ -292,7 +292,7 @@ jobs:
           df -h
           #sudo apt install -y xvfb
           #xvfb-run ls
-          curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
+          curl -L https://github.com/docker/compose/releases/download/v${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
           chmod +x docker-compose
           sudo mv docker-compose /usr/local/bin
           IP=`hostname -I | awk '{print $1}'`

diff --git a/.github/workflows/nightly-trivy-scan.yml b/.github/workflows/nightly-trivy-scan.yml
@@ -12,7 +12,7 @@ jobs:
       matrix:
         # maintain the versions of harbor that need to be actively
         # security scanned
-        versions: [dev, v2.10.0-dev]
+        versions: [dev, v2.11.0-dev]
         # list of images that need to be scanned
         images: [harbor-core, harbor-db, harbor-exporter, harbor-jobservice, harbor-log, harbor-portal, harbor-registryctl, prepare]
     permissions:

diff --git a/RELEASES.md b/RELEASES.md
@@ -16,9 +16,9 @@ Patch releases are based on the major/minor release branch, the release cadence
 ### Minor Release Support Matrix
 | Version        | Supported          |
 |----------------| ------------------ |
+| Harbor v2.11.x | :white_check_mark: |
 | Harbor v2.10.x | :white_check_mark: |
 | Harbor v2.9.x  | :white_check_mark: |
-| Harbor v2.8.x  | :white_check_mark: |
 
 ### Upgrade path and support policy
 The upgrade path for Harbor is (1) 2.2.x patch releases are always compatible with its major and minor version. For example, previous released 2.2.x can be upgraded to most recent 2.2.3 release. (2) Harbor only supports two previous minor releases to upgrade to current minor release. For example, 2.3.0 will only support 2.1.0 and 2.2.0 to upgrade from, 2.0.0 to 2.3.0 is not supported. One should upgrade to 2.2.0 first, then to 2.3.0.

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-v2.11.0
+v2.12.0
diff --git a/src/controller/proxy/controller.go b/src/controller/proxy/controller.go
@@ -172,9 +172,6 @@ func (c *controller) UseLocalManifest(ctx context.Context, art lib.ArtifactInfo,
 		return false, nil, err
 	}
 	if !exist || desc == nil {
-		go func() {
-			c.local.DeleteManifest(remoteRepo, art.Tag)
-		}()
 		return false, nil, errors.NotFoundError(fmt.Errorf("repo %v, tag %v not found", art.Repository, art.Tag))
 	}
 
@@ -220,11 +217,6 @@ func (c *controller) ProxyManifest(ctx context.Context, art lib.ArtifactInfo, re
 	ref := getReference(art)
 	man, dig, err := remote.Manifest(remoteRepo, ref)
 	if err != nil {
-		if errors.IsNotFoundErr(err) {
-			go func() {
-				c.local.DeleteManifest(remoteRepo, art.Tag)
-			}()
-		}
 		return man, err
 	}
 	ct, _, err := man.Payload()

diff --git a/src/controller/scan/base_controller.go b/src/controller/scan/base_controller.go
@@ -333,7 +333,11 @@ func (bc *basicController) Scan(ctx context.Context, artifact *ar.Artifact, opti
 		if op := operator.FromContext(ctx); op != "" {
 			extraAttrs["operator"] = op
 		}
-		executionID, err := bc.execMgr.Create(ctx, job.ImageScanJobVendorType, artifact.ID, task.ExecutionTriggerManual, extraAttrs)
+		vendorType := handler.JobVendorType()
+		// for vulnerability and generate sbom, use different vendor type
+		// because the execution reaper only keep the latest execution for the vendor type IMAGE_SCAN
+		// both vulnerability and sbom need to keep the latest scan execution to get the latest scan status
+		executionID, err := bc.execMgr.Create(ctx, vendorType, artifact.ID, task.ExecutionTriggerManual, extraAttrs)
 		if err != nil {
 			return err
 		}
@@ -364,7 +368,8 @@ func (bc *basicController) Stop(ctx context.Context, artifact *ar.Artifact, capT
 	if artifact == nil {
 		return errors.New("nil artifact to stop scan")
 	}
-	query := q.New(q.KeyWords{"vendor_type": job.ImageScanJobVendorType, "extra_attrs.artifact.digest": artifact.Digest, "extra_attrs.enabled_capabilities.type": capType})
+	vendorType := sca.GetScanHandler(capType).JobVendorType()
+	query := q.New(q.KeyWords{"vendor_type": vendorType, "extra_attrs.artifact.digest": artifact.Digest, "extra_attrs.enabled_capabilities.type": capType})
 	executions, err := bc.execMgr.List(ctx, query)
 	if err != nil {
 		return err
@@ -960,7 +965,8 @@ func (bc *basicController) launchScanJob(ctx context.Context, param *launchScanJ
 	params[sca.JobParameterRequest] = sJSON
 	params[sca.JobParameterMimes] = mimes
 	params[sca.JobParameterRobot] = robotJSON
-
+	// because there is only one task type implementation
+	// both the vulnerability scan and generate sbom use the same job type for now
 	j := &task.Job{
 		Name: job.ImageScanJobVendorType,
 		Metadata: &job.Metadata{

diff --git a/src/controller/scan/base_controller_test.go b/src/controller/scan/base_controller_test.go
@@ -342,6 +342,8 @@ func (suite *ControllerTestSuite) SetupSuite() {
 		reportConverter: &postprocessorstesting.ScanReportV1ToV2Converter{},
 		cache:           func() cache.Cache { return suite.cache },
 	}
+	mock.OnAnything(suite.scanHandler, "JobVendorType").Return("IMAGE_SCAN")
+
 }
 
 // TearDownSuite ...

diff --git a/src/controller/task/sweep.go b/src/controller/task/sweep.go
@@ -41,8 +41,8 @@ const (
 	systemVendorID = -1
 
 	cronTypeCustom = "Custom"
-	// run for every hour
-	cronSpec = "0 0 * * * *"
+	// run for every day
+	cronSpec = "0 0 0 * * *"
 )
 
 func init() {

diff --git a/src/go.mod b/src/go.mod
@@ -7,26 +7,26 @@ require (
 	github.com/Masterminds/semver v1.5.0
 	github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20190726115642-cd293c93fd97
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2
-	github.com/aws/aws-sdk-go v1.50.24
-	github.com/beego/beego/v2 v2.0.6
+	github.com/aws/aws-sdk-go v1.53.14
+	github.com/beego/beego/v2 v2.2.1
 	github.com/beego/i18n v0.0.0-20140604031826-e87155e8f0c0
 	github.com/bmatcuk/doublestar v1.3.4
 	github.com/casbin/casbin v1.9.1
 	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/cloudevents/sdk-go/v2 v2.15.2
 	github.com/coreos/go-oidc/v3 v3.10.0
 	github.com/dghubble/sling v1.1.0
-	github.com/docker/distribution v2.8.2+incompatible
+	github.com/docker/distribution v2.8.3+incompatible
 	github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7
-	github.com/go-asn1-ber/asn1-ber v1.5.6
+	github.com/go-asn1-ber/asn1-ber v1.5.7
 	github.com/go-ldap/ldap/v3 v3.4.6
 	github.com/go-openapi/errors v0.22.0
-	github.com/go-openapi/loads v0.21.2 // indirect
+	github.com/go-openapi/loads v0.21.2
 	github.com/go-openapi/runtime v0.26.2
-	github.com/go-openapi/spec v0.20.11 // indirect
+	github.com/go-openapi/spec v0.20.11
 	github.com/go-openapi/strfmt v0.23.0
 	github.com/go-openapi/swag v0.23.0
-	github.com/go-openapi/validate v0.22.3 // indirect
+	github.com/go-openapi/validate v0.22.3
 	github.com/go-redis/redis/v8 v8.11.4
 	github.com/gocarina/gocsv v0.0.0-20210516172204-ca9e8a8ddea8
 	github.com/gocraft/work v0.5.1
@@ -54,27 +54,27 @@ require (
 	github.com/stretchr/testify v1.9.0
 	github.com/tencentcloud/tencentcloud-sdk-go v3.0.233+incompatible
 	github.com/vmihailenco/msgpack/v5 v5.4.1
-	github.com/volcengine/volcengine-go-sdk v1.0.97
+	github.com/volcengine/volcengine-go-sdk v1.0.138
 	go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.51.0
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0
-	go.opentelemetry.io/otel v1.26.0
+	go.opentelemetry.io/otel v1.27.0
 	go.opentelemetry.io/otel/exporters/jaeger v1.0.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0
 	go.opentelemetry.io/otel/sdk v1.26.0
-	go.opentelemetry.io/otel/trace v1.26.0
+	go.opentelemetry.io/otel/trace v1.27.0
 	go.uber.org/ratelimit v0.3.1
-	golang.org/x/crypto v0.23.0
-	golang.org/x/net v0.25.0
+	golang.org/x/crypto v0.24.0
+	golang.org/x/net v0.26.0
 	golang.org/x/oauth2 v0.19.0
-	golang.org/x/sync v0.6.0
-	golang.org/x/text v0.15.0
+	golang.org/x/sync v0.7.0
+	golang.org/x/text v0.16.0
 	golang.org/x/time v0.5.0
 	gopkg.in/h2non/gock.v1 v1.1.2
 	gopkg.in/yaml.v2 v2.4.0
-	helm.sh/helm/v3 v3.14.4
+	helm.sh/helm/v3 v3.15.2
 	k8s.io/api v0.30.0
 	k8s.io/apimachinery v0.30.0
-	k8s.io/client-go v0.29.0
+	k8s.io/client-go v0.30.0
 	sigs.k8s.io/yaml v1.4.0
 )
 
@@ -101,8 +101,8 @@ require (
 	github.com/denverdino/aliyungo v0.0.0-20191227032621-df38c6fa730c // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/dnaeon/go-vcr v1.2.0 // indirect
-	github.com/docker/cli v24.0.6+incompatible // indirect
-	github.com/docker/docker v24.0.9+incompatible // indirect
+	github.com/docker/cli v25.0.1+incompatible // indirect
+	github.com/docker/docker v25.0.5+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.7.0 // indirect
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
@@ -160,18 +160,19 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/vbatts/tar-split v0.11.3 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/volcengine/volc-sdk-golang v1.0.23 // indirect
 	go.mongodb.org/mongo-driver v1.14.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 // indirect
-	go.opentelemetry.io/otel/metric v1.26.0 // indirect
+	go.opentelemetry.io/otel/metric v1.27.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.2.0 // indirect
-	go.uber.org/atomic v1.7.0 // indirect
-	go.uber.org/multierr v1.6.0 // indirect
-	go.uber.org/zap v1.19.0 // indirect
-	golang.org/x/sys v0.20.0 // indirect
-	golang.org/x/term v0.20.0 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/multierr v1.7.0 // indirect
+	go.uber.org/zap v1.19.1 // indirect
+	golang.org/x/sys v0.21.0 // indirect
+	golang.org/x/term v0.21.0 // indirect
 	google.golang.org/api v0.162.0 // indirect
 	google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect