Upgrade node dependencies and node to 22

.github/actions/ft-test/action.yaml

@@ -17,6 +17,10 @@ inputs:
     description: name to the runner script
     required: false
     default: run_ft_tests.bash
+  cloudserver_image:
+    description: name of the cloudserver image
+    required: false
+    default: "ghcr.io/scality/cloudserver:8.6.8"
 
 runs:
   using: composite
@@ -26,6 +30,7 @@ runs:
       run: .github/scripts/${{ inputs.runner }} ft_test:${{ inputs.testsuite }}
       env:
         BACKBEAT_CONFIG_FILE: ${{ inputs.config }}
+        CLOUDSERVER_IMAGE: ${{ inputs.cloudserver_image }}
 
     - uses: codecov/codecov-action@v4
       with:

.github/dockerfiles/mongodb/conf/initReplicaSet.js

@@ -1,4 +1,6 @@
+/* global rs */ // rs is a built-in variable in the mongo shell
+
 rs.initiate({
-    _id: "rs0",
-    members: [{ _id: 0, host: "127.0.0.1:27018" }]
+    _id: 'rs0',
+    members: [{ _id: 0, host: '127.0.0.1:27018' }]
 });

.github/dockerfiles/syntheticbucketd/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:16
+FROM node:22
 
 RUN mkdir /app
 COPY package.json /app

.github/scripts/run_ft_tests.bash

@@ -7,18 +7,13 @@ NODE_PATH=${NODE_PATH:-node_modules}
 # port for cloudserver
 PORT=8000
 
-if [ ! -d "${NODE_PATH}/@zenko/cloudserver" ]; then
-    echo "cloudserver module was not found!"
-    exit 1
-fi
-
 trap killandsleep EXIT
 
 killandsleep () {
   kill -9 $(lsof -t -i:$PORT) || true
   sleep 10
 }
 
-cd ${NODE_PATH}/@zenko/cloudserver && yarn run mem_backend & bash tests/utils/wait_for_local_port.bash $PORT 40
+docker run -p $PORT:8000 -d $CLOUDSERVER_IMAGE /bin/sh -c "yarn run mem_backend" & bash tests/utils/wait_for_local_port.bash $PORT 40
 ./node_modules/.bin/nyc --clean --silent yarn run $1
 ./node_modules/.bin/nyc report --report-dir "./coverage/$1" --reporter=lcov

.github/scripts/run_server_tests.bash

@@ -11,8 +11,8 @@ killandsleep () {
   sleep 10
 }
 
-./node_modules/.bin/nyc --clean --silent yarn start &
-bash tests/utils/wait_for_local_port.bash $PORT 40
+./node_modules/.bin/nyc --silent --clean yarn start &
+bash tests/utils/wait_for_local_port.bash $PORT 120
 yarn run $1
 killandsleep
 ./node_modules/.bin/nyc report --report-dir "./coverage/$1" --reporter=lcov

.github/workflows/docker-build.yaml

@@ -17,6 +17,11 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+          cache: yarn
+
       - name: Set up Docker Buildk
         uses: docker/setup-buildx-action@v3
 

.github/workflows/release.yaml

@@ -19,6 +19,11 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+          cache: yarn
+
       # TODO: remove the following step once Oras CLI 0.13.0 bug https://github.com/oras-project/oras/issues/447 is fixed.
       - name: Downgrade Oras to 0.12.0
         run: |

.github/workflows/tests.yaml

@@ -2,7 +2,7 @@
 name: tests
 
 env:
-  GINKGO_VERSION: v1.15.2
+  GINKGO_VERSION: v2.21.0
 
 on:
   push:
@@ -12,7 +12,7 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04-2core
     permissions:
       # Need to explicitely add package write permissions for dependabot
       contents: read
@@ -21,6 +21,11 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v4
 
+    - uses: actions/setup-node@v4
+      with:
+        node-version: '22'
+        cache: yarn
+
     - name: Set up Docker Buildk
       uses: docker/setup-buildx-action@v3
 
@@ -60,24 +65,29 @@ jobs:
         cache-to: type=gha,mode=max,scope=ci-mongodb
 
   bucket-scanner:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04-2core
     steps:
     - name: Checkout
       uses: actions/checkout@v4
 
+    - uses: actions/setup-node@v4
+      with:
+        node-version: '22'
+        cache: yarn
+
     - uses: actions/setup-go@v5
       with:
-        go-version: '1.16.2'
+        go-version: '1.23.0'
         cache-dependency-path: bucket-scanner/go.sum
 
     - name: Install ginkgo
-      run: go get github.com/onsi/ginkgo/ginkgo@${GINKGO_VERSION}
+      run: go install github.com/onsi/ginkgo/v2/ginkgo@${GINKGO_VERSION}
 
     - name: Run bucket scanner unit tests
       run: >-
-        ginkgo -r --randomizeAllSpecs --randomizeSuites --failOnPending --progress   \
+        ginkgo -r --randomize-suites --fail-on-pending --show-node-events   \
                --cover --trace --race --covermode=atomic --coverprofile=coverage.out \
-               -nodes 1 -stream -timeout 5m -slowSpecThreshold 60
+               -nodes 1 -timeout 5m
       working-directory: bucket-scanner
 
     - uses: codecov/codecov-action@v4
@@ -88,7 +98,7 @@ jobs:
 
   tests:
     needs: build
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04-2core
     services:
       redis:
         image: redis:alpine
@@ -120,7 +130,7 @@ jobs:
       uses: actions/checkout@v4
     - uses: actions/setup-node@v4
       with:
-        node-version: '16'
+        node-version: '22'
         cache: yarn
     - name: Install node dependencies
       run: yarn install --ignore-engines --frozen-lockfile --network-concurrency 1
@@ -192,3 +202,9 @@ jobs:
         # will crash if they end up in memory all at the same time (circuit breaking
         # ineffective) while waiting to be committed to the kafka topic.
         NODE_OPTIONS: '--max-old-space-size=150'
+
+    - name: Move S3 and IAM logs and data to artifacts directory
+      run: |-
+        set -e -o pipefail;
+        mkdir -p .github/e2e_tests/artifacts/logs .github/e2e_tests/artifacts/data
+      if: always()

Dockerfile

@@ -1,6 +1,6 @@
-ARG NODE_VERSION=16.20-bullseye-slim
+ARG NODE_VERSION=22.11.0-bullseye-slim
 
-FROM node:${NODE_VERSION} as builder
+FROM node:${NODE_VERSION} AS builder
 
 WORKDIR /usr/src/app
 
@@ -22,7 +22,7 @@ RUN apt-get update \
         libffi-dev \
         libzstd-dev
 
-ENV DOCKERIZE_VERSION v0.6.1
+ENV DOCKERIZE_VERSION=v0.8.0
 
 RUN wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
     && tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \

README.md

@@ -374,6 +374,3 @@ $HOME
     ├── cloudserver
     └── vault
 ```
-
-[badgepriv]: http://ci.ironmann.io/gh/scality/backbeat.svg?style=svg&circle-token=32e5dfd968e673450c44f0a255d1a812bae9b00c
-[badgepub]: https://circleci.com/gh/scality/backbeat.svg?style=svg

bin/backbeat.js

@@ -1,4 +1,4 @@
-'use strict'; // eslint-disable-line strict
+'use strict';  
 
 const werelogs = require('werelogs');
 

bucket-scanner/coverage.out

@@ -0,0 +1,109 @@
+mode: atomic
+github.com/scality/backbeat/bucket-scanner/pkg/counter/base_counter.go:15.36,17.2 1 3
+github.com/scality/backbeat/bucket-scanner/pkg/counter/base_counter.go:19.51,21.2 1 1000000
+github.com/scality/backbeat/bucket-scanner/pkg/counter/base_counter.go:23.51,25.2 1 0
+github.com/scality/backbeat/bucket-scanner/pkg/counter/base_counter.go:27.35,29.2 1 3
+github.com/scality/backbeat/bucket-scanner/pkg/counter/base_counter.go:31.36,33.2 1 2
+github.com/scality/backbeat/bucket-scanner/pkg/counter/base_counter.go:35.39,37.2 1 1
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/by_status_counter_set.go:31.61,42.2 1 58
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/by_status_counter_set.go:44.67,51.2 6 3
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/by_status_counter_set.go:53.64,60.2 6 3
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/by_status_counter_set.go:62.41,69.2 6 2
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/by_status_counter_set.go:71.75,80.2 1 7
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/by_status_counter_set.go:82.81,89.2 6 3
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/by_status_counter_set.go:91.134,92.16 1 32
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/by_status_counter_set.go:93.40,94.19 1 8
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/by_status_counter_set.go:94.19,96.4 1 3
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/by_status_counter_set.go:97.3,98.92 2 8
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/by_status_counter_set.go:100.38,101.29 1 10
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/by_status_counter_set.go:103.37,104.19 1 6
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/by_status_counter_set.go:104.19,106.4 1 3
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/by_status_counter_set.go:107.3,107.28 1 6
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/by_status_counter_set.go:109.38,110.29 1 3
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/by_status_counter_set.go:112.41,113.32 1 2
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/by_status_counter_set.go:115.10,116.30 1 3
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/by_status_counter_set.go:120.71,129.2 1 1
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/counter_set.go:24.35,31.2 1 17
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/counter_set.go:33.49,40.40 5 1
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/counter_set.go:40.40,44.3 3 1
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/counter_set.go:47.46,53.43 4 1
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/counter_set.go:53.43,56.3 2 1
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/counter_set.go:59.31,65.2 4 1
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/counter_set.go:67.86,72.2 3 25
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/counter_set.go:74.86,76.9 2 26
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/counter_set.go:76.9,79.3 2 22
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/counter_set.go:81.2,81.13 1 26
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/counter_set.go:84.57,90.50 4 1
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/counter_set.go:90.50,92.3 1 1
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/counter_set.go:94.2,97.3 1 1
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/counter_set.go:100.63,103.33 2 1
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/counter_set.go:103.33,106.3 2 1
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/counter_set.go:113.127,116.39 2 8
+github.com/scality/backbeat/bucket-scanner/pkg/counter/replication/counter_set.go:116.39,118.3 1 16
+github.com/scality/backbeat/bucket-scanner/pkg/counter/tracking_counter.go:11.44,15.2 1 0
+github.com/scality/backbeat/bucket-scanner/pkg/exporter/exporter.go:46.30,50.2 1 10
+github.com/scality/backbeat/bucket-scanner/pkg/exporter/exporter.go:52.50,85.2 2 8
+github.com/scality/backbeat/bucket-scanner/pkg/exporter/exporter.go:87.85,94.2 6 22
+github.com/scality/backbeat/bucket-scanner/pkg/exporter/exporter.go:97.73,98.20 1 7
+github.com/scality/backbeat/bucket-scanner/pkg/exporter/exporter.go:98.20,100.3 1 5
+github.com/scality/backbeat/bucket-scanner/pkg/exporter/exporter.go:102.2,102.16 1 7
+github.com/scality/backbeat/bucket-scanner/pkg/exporter/exporter.go:105.79,107.24 2 15
+github.com/scality/backbeat/bucket-scanner/pkg/exporter/exporter.go:107.24,111.3 3 20
+github.com/scality/backbeat/bucket-scanner/pkg/exporter/exporter.go:113.2,115.43 2 15
+github.com/scality/backbeat/bucket-scanner/pkg/exporter/exporter.go:115.43,118.3 2 7
+github.com/scality/backbeat/bucket-scanner/pkg/exporter/exporter.go:121.153,124.6 2 7
+github.com/scality/backbeat/bucket-scanner/pkg/exporter/exporter.go:124.6,125.10 1 19
+github.com/scality/backbeat/bucket-scanner/pkg/exporter/exporter.go:126.21,127.10 1 5
+github.com/scality/backbeat/bucket-scanner/pkg/exporter/exporter.go:129.26,130.11 1 14
+github.com/scality/backbeat/bucket-scanner/pkg/exporter/exporter.go:130.11,132.5 1 2
+github.com/scality/backbeat/bucket-scanner/pkg/exporter/exporter.go:134.4,136.56 2 12
+github.com/scality/backbeat/bucket-scanner/pkg/exporter/exporter.go:142.164,149.12 3 4
+github.com/scality/backbeat/bucket-scanner/pkg/exporter/exporter.go:149.12,154.3 3 4
+github.com/scality/backbeat/bucket-scanner/pkg/exporter/exporter.go:158.2,165.12 5 4
+github.com/scality/backbeat/bucket-scanner/pkg/exporter/exporter.go:165.12,169.58 3 4
+github.com/scality/backbeat/bucket-scanner/pkg/exporter/exporter.go:169.58,171.4 1 0
+github.com/scality/backbeat/bucket-scanner/pkg/exporter/exporter.go:176.2,178.12 2 4
+github.com/scality/backbeat/bucket-scanner/pkg/exporter/exporter.go:178.12,187.47 6 4
+github.com/scality/backbeat/bucket-scanner/pkg/exporter/exporter.go:187.47,189.4 1 0
+github.com/scality/backbeat/bucket-scanner/pkg/exporter/exporter.go:194.2,196.80 2 4
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:65.38,70.2 4 5
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:72.69,74.16 2 0
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:74.16,76.3 1 0
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:78.2,78.85 1 0
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:89.55,90.30 1 0
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:90.30,91.21 1 0
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:94.2,94.33 1 0
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:94.33,95.24 1 0
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:98.2,98.140 1 0
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:101.63,102.30 1 1
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:102.30,103.21 1 0
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:106.2,106.33 1 1
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:106.33,107.24 1 0
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:110.2,111.24 2 1
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:114.87,118.16 3 6
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:118.16,120.3 1 0
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:122.2,125.16 3 6
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:128.91,130.2 1 2
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:140.63,143.2 1 0
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:145.54,148.2 2 1
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:150.69,154.16 3 2
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:154.16,156.3 1 0
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:158.2,158.25 1 2
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:158.25,160.17 2 2
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:160.17,162.4 1 0
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:164.3,165.21 2 2
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:168.2,168.15 1 2
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:177.41,181.2 2 0
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:183.49,188.2 3 1
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:190.59,194.16 3 2
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:194.16,196.3 1 0
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:198.2,198.15 1 2
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:208.41,210.2 1 0
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:212.49,215.2 2 1
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:217.59,221.16 3 2
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:221.16,223.3 1 0
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:225.2,225.25 1 2
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:225.25,227.17 2 2
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:227.17,229.4 1 0
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:231.3,232.21 2 2
+github.com/scality/backbeat/bucket-scanner/pkg/types/types.go:235.2,235.15 1 2

bucket-scanner/go.mod

@@ -1,16 +1,34 @@
 module github.com/scality/backbeat/bucket-scanner
 
-go 1.15
+go 1.23.2
 
 require (
-	github.com/confluentinc/confluent-kafka-go v1.6.1
 	github.com/go-resty/resty/v2 v2.5.0
 	github.com/gosuri/uilive v0.0.4
-	github.com/onsi/ginkgo v1.15.1
-	github.com/onsi/gomega v1.11.0
+	github.com/onsi/ginkgo/v2 v2.21.0
+	github.com/onsi/gomega v1.34.2
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.9.0
+	github.com/prometheus/client_model v0.2.0
 	github.com/sirupsen/logrus v1.8.1
-	github.com/spf13/cobra v1.1.3
-	github.com/spf13/viper v1.7.1
+)
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect
+	github.com/mattn/go-isatty v0.0.4 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/prometheus/common v0.15.0 // indirect
+	github.com/prometheus/procfs v0.2.0 // indirect
+	golang.org/x/net v0.30.0 // indirect
+	golang.org/x/sys v0.26.0 // indirect
+	golang.org/x/text v0.19.0 // indirect
+	golang.org/x/tools v0.26.0 // indirect
+	google.golang.org/protobuf v1.34.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )