BB-295 - bootstrap ts compilation

.eslintrc

@@ -1,8 +1,15 @@
 {
-    "extends": "scality",
+    "extends": [
+        "scality",
+        "plugin:jest/recommended"
+    ],
     "parserOptions": {
         "ecmaVersion": 2020
     },
+    "plugins": [
+        "jest",
+        "mocha"
+    ],
     "rules": {
         "object-curly-newline": "off",
         "import/newline-after-import": "off",
@@ -33,13 +40,40 @@
         "space-unary-ops": "off",
         "no-undef-init": "off",
         "newline-per-chained-call": "off",
-        "no-useless-escape": "off"
+        "no-useless-escape": "off",
+        "mocha/no-exclusive-tests": "error",
+        "jest/no-done-callback": "off",
+        "jest/expect-expect": [
+            "warn",
+            {
+                "assertFunctionNames": [
+                    "expect",
+                    "assert",
+                    "assert.*",
+                    "_assertCredentials"
+                ]
+            }
+        ],
+        "import/extensions": [
+            "error",
+            "ignorePackages",
+            {
+                "js": "never",
+                "jsx": "never",
+                "ts": "never",
+                "tsx": "never"
+            }
+        ]
+    },
+    "env": {
+        "jest/globals": true
     },
     "settings": {
         "import/resolver": {
-          "node": {
-            "paths": ["/backbeat/node_modules", "node_modules"]
-          }
+            "node": {
+                "paths": ["/backbeat/node_modules", "node_modules"],
+                "extensions": [".js", ".jsx", ".ts", ".tsx"]
+            }
         }
-      }
+    }
 }

.github/dockerfiles/mongodb/Dockerfile

@@ -1,4 +1,4 @@
-FROM mongo:4.2.24
+FROM mongo:4.4.21
 
 ENV USER=scality \
     HOME_DIR=/home/scality \

.github/workflows/tests.yaml

@@ -44,8 +44,7 @@ jobs:
       uses: docker/build-push-action@v4
       with:
         push: true
-        context: .
-        file: .github/dockerfiles/syntheticbucketd/Dockerfile
+        context: tests/utils/syntheticbucketd
         tags: "ghcr.io/scality/backbeat/syntheticbucketd:${{ github.sha }}"
         cache-from: type=gha,scope=syntheticbucketd
         cache-to: type=gha,mode=max,scope=syntheticbucketd
@@ -105,6 +104,8 @@ jobs:
         cache: yarn
     - name: Install node dependencies
       run: yarn install --ignore-engines --frozen-lockfile --network-concurrency 1
+    - name: Compile TypeScript
+      run: yarn build
     - name: Install ginkgo
       run: go get github.com/onsi/ginkgo/ginkgo@${GINKGO_VERSION}
     - name: Lint markdown
@@ -147,11 +148,10 @@ jobs:
         BACKBEAT_CONFIG_FILE: "tests/config.json"
     - name: run backbeat notification feature tests
       run: yarn run ft_test:notification
-
     - name: run ballooning tests for lifecycle conductor
-      run: yarn mocha tests/performance/lifecycle/conductor-check-memory-balloon.js
+      run: .github/scripts/run_ft_tests.bash perf_test:lifecycle
       env:
-        # Constrain heap long-lived heap size to 150MB, so that pushing 200K messages
+        # Constrain heap long-lived heap size to 200MB, so that pushing 200K messages
         # will crash if they end up in memory all at the same time (circuit breaking
         # ineffective) while waiting to be committed to the kafka topic.
-        NODE_OPTIONS: '--max-old-space-size=150'
+        NODE_OPTIONS: '--max-old-space-size=200'

.gitignore

@@ -15,3 +15,6 @@ node_modules
 
 # Redis
 *.rdb
+
+# TypeScript compilation output
+dist/

Dockerfile

@@ -29,24 +29,30 @@ RUN wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSI
     && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz
 
 COPY package.json yarn.lock /usr/src/app/
-RUN yarn install --ignore-engines --frozen-lockfile --production --network-concurrency 1 \
-    && rm -rf /var/lib/apt/lists/* \
-    && rm -rf ~/.node-gyp \
-    && rm -rf /tmp/yarn-*
+
+RUN yarn install --ignore-engines --frozen-lockfile --network-concurrency 1
+
+COPY . /usr/src/app
+
+RUN yarn build
+
+RUN rm -rf node_modules \
+    && yarn install --production --frozen-lockfile --ignore-optional
 
 ################################################################################
 FROM node:${NODE_VERSION}
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
         jq \
         tini \
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /usr/src/app
 
 # Keep the .git directory in order to properly report version
-COPY . /usr/src/app
+COPY ./package.json ./docker-entrypoint.sh ./
+COPY --from=builder /usr/src/app/dist ./
 COPY --from=builder /usr/src/app/node_modules ./node_modules/
 COPY --from=builder /usr/local/bin/dockerize /usr/local/bin/
 

extensions/lifecycle/tasks/LifecycleUpdateExpirationTask.js

@@ -4,7 +4,6 @@ const { errors } = require('arsenal');
 const ObjectMD = require('arsenal').models.ObjectMD;
 const BackbeatTask = require('../../../lib/tasks/BackbeatTask');
 const ActionQueueEntry = require('../../../lib/models/ActionQueueEntry');
-const locations = require('../../../conf/locationConfig.json') || {};
 
 class LifecycleUpdateExpirationTask extends BackbeatTask {
     /**
@@ -144,13 +143,20 @@ class LifecycleUpdateExpirationTask extends BackbeatTask {
         });
 
         async.waterfall([
-            next => this._getMetadata(entry, log, next),
-            (objMD, next) => {
-                const coldLocation = entry.getAttribute('target.location') ||
-                    // If location not specified, use the first (and only) location
-                    // This is a temporary fix, until Sorbet is fixed to provide the information
-                    Object.keys(locations).find(name => locations[name].isCold);
-
+            next => {
+                const coldLocation = entry.getAttribute('target.location');
+                if (!coldLocation) {
+                    // this should never happen as sorbet always sets the location attribute
+                    log.error('missing target location', {
+                        entry: entry.getLogInfo(),
+                        method: 'LifecycleUpdateExpirationTask.processActionEntry',
+                    });
+                    return next(errors.MissingParameter.customizeDescription('missing target location'));
+                }
+                return next(null, coldLocation);
+            },
+            (coldLocation, next) => this._getMetadata(entry, log, (err, objMD) => next(err, coldLocation, objMD)),
+            (coldLocation, objMD, next) => {
                 const archive = objMD.getArchive();
 
                 // Confirm the object has indeed expired: it can happen that the

extensions/replication/queueProcessor/QueueProcessor.js

@@ -459,18 +459,18 @@ class QueueProcessor extends EventEmitter {
      */
     _setupRedis(redisConfig) {
         // redis pub/sub for pause/resume
-        const redis = new Redis(redisConfig);
+        this._redis = new Redis(redisConfig);
         // redis subscribe to site specific channel
         const channelName = `${this.repConfig.topic}-${this.site}`;
-        redis.subscribe(channelName, err => {
+        this._redis.subscribe(channelName, err => {
             if (err) {
                 this.logger.fatal('queue processor failed to subscribe to ' +
                                   `crr redis channel for location ${this.site}`,
                                   { method: 'QueueProcessor.constructor',
                                     error: err });
                 process.exit(1);
             }
-            redis.on('message', (channel, message) => {
+            this._redis.on('message', (channel, message) => {
                 const validActions = {
                     pauseService: this._pauseService.bind(this),
                     resumeService: this._resumeService.bind(this),

extensions/replication/tasks/ReplicateObject.js

@@ -8,7 +8,7 @@ const ObjectMDLocation = require('arsenal').models.ObjectMDLocation;
 const BackbeatClient = require('../../../lib/clients/BackbeatClient');
 const BackbeatMetadataProxy = require('../../../lib/BackbeatMetadataProxy');
 
-const mapLimitWaitPendingIfError = require('../../../lib/util/mapLimitWaitPendingIfError');
+const mapLimitWaitPendingIfError = require('../../../lib/util/mapLimitWaitPendingIfError').default;
 const { attachReqUids, TIMEOUT_MS } = require('../../../lib/clients/utils');
 const getExtMetrics = require('../utils/getExtMetrics');
 const BackbeatTask = require('../../../lib/tasks/BackbeatTask');

lib/BackbeatConsumer.js

@@ -8,7 +8,7 @@ const Logger = require('werelogs').Logger;
 const { BreakerState, CircuitBreaker } = require('breakbeat').CircuitBreaker;
 
 const BackbeatProducer = require('./BackbeatProducer');
-const OffsetLedger = require('./OffsetLedger');
+const OffsetLedger = require('./OffsetLedger').default;
 const KafkaBacklogMetrics = require('./KafkaBacklogMetrics');
 const {
     startCircuitBreakerMetricsExport,

lib/OffsetLedger.js → lib/OffsetLedger.ts

@@ -1,4 +1,11 @@
-const assert = require('assert');
+import assert from 'assert';
+
+type PartitionOffsets = {
+    processing: number[],
+    latestConsumed: number | null
+};
+
+type TopicOffsets = { [key: number]: PartitionOffsets}
 
 /**
  * @class OffsetLedger
@@ -8,16 +15,20 @@ const assert = require('assert');
  * when messages are processed asynchronously and processing may end
  * out-of-order.
  */
-class OffsetLedger {
+
+ export default class OffsetLedger {
+
+    #ledger: { [topic: string]: TopicOffsets };
+
     constructor() {
-        this._ledger = {};
+        this.#ledger = {};
     }
 
-    _getPartitionOffsets(topic, partition) {
-        let topicOffsets = this._ledger[topic];
+    _getPartitionOffsets(topic: string, partition: number) {
+        let topicOffsets = this.#ledger[topic];
         if (!topicOffsets) {
             topicOffsets = {};
-            this._ledger[topic] = topicOffsets;
+            this.#ledger[topic] = topicOffsets;
         }
         let partitionOffsets = topicOffsets[partition];
         if (!partitionOffsets) {
@@ -30,7 +41,7 @@ class OffsetLedger {
         return partitionOffsets;
     }
 
-    _getPartitionCommittableOffset(partitionOffsets) {
+    _getPartitionCommittableOffset(partitionOffsets: PartitionOffsets): number | null {
         // we can commit up to the lowest offset still being processed
         // since it means all lower offsets have already been
         // processed. If nothing is being processed, the latest
@@ -49,13 +60,8 @@ class OffsetLedger {
     /**
      * Function to be called as soon as a new message is received from
      * a Kafka topic and about to start being processed.
-     *
-     * @param {string} topic - topic name
-     * @param {number} partition - partition number
-     * @param {number} offset - offset of consumed message
-     * @return {undefined}
      */
-    onOffsetConsumed(topic, partition, offset) {
+    onOffsetConsumed(topic: string, partition: number, offset: number): undefined {
         // make sure offset is a positive number not to jeopardize
         // processing sanity
         assert(Number.isInteger(offset) && offset >= 0);
@@ -78,13 +84,10 @@ class OffsetLedger {
     /**
      * Function to be called when a message is completely processed.
      *
-     * @param {string} topic - topic name
-     * @param {number} partition - partition number
-     * @param {number} offset - offset of processed message
-     * @return {number} - highest committable offset for this
-     * topic/partition (as returned by getCommittableOffset())
+     * @return highest committable offset for this topic/partition
+     * (as returned by getCommittableOffset())
      */
-    onOffsetProcessed(topic, partition, offset) {
+    onOffsetProcessed(topic: string, partition: number, offset: number): number | null {
         const partitionOffsets = this._getPartitionOffsets(topic, partition);
         partitionOffsets.processing =
             partitionOffsets.processing.filter(pOff => pOff !== offset);
@@ -94,12 +97,9 @@ class OffsetLedger {
     /**
      * Get the highest committable offset for a topic/partition
      *
-     * @param {string} topic - topic name
-     * @param {number} partition - partition number
-     * @param {number} offset - offset of processed message
-     * @return {number} - highest committable offset for this topic/partition
+     * @return highest committable offset for this topic/partition
      */
-    getCommittableOffset(topic, partition) {
+    getCommittableOffset(topic: string, partition: number): number | null {
         const partitionOffsets = this._getPartitionOffsets(topic, partition);
         return this._getPartitionCommittableOffset(partitionOffsets);
     }
@@ -108,28 +108,25 @@ class OffsetLedger {
      * Get how many entries have been consumed but not yet fully
      * processed/committable
      *
-     * @param {string} [topic] - topic name
-     * @param {number} [partition] - partition number
-     * @return {number} - number of consumed but not committable
-     * entries for this topic/partition, or for this topic (if no
-     * partition given), or for all topics and partitions (if none of
-     * topic and partition is provided)
+     * @return number of consumed but not committable entries for this
+     * topic/partition, or for this topic (if no partition given), or for
+     * all topics and partitions (if none of topic and partition is provided)
      */
-    getProcessingCount(topic, partition) {
-        if (topic && !this._ledger[topic]) {
+    getProcessingCount(topic?: string, partition?: number): number {
+        if (topic && !this.#ledger[topic]) {
             return 0;
         }
         if (topic && partition !== undefined &&
-            !this._ledger[topic][partition]) {
+            !this.#ledger[topic][partition]) {
             return 0;
         }
         let count = 0;
-        const topics = topic ? [topic] : Object.keys(this._ledger);
+        const topics = topic ? [topic] : Object.keys(this.#ledger);
         topics.forEach(t => {
             const partitions = partition !== undefined ?
-                  [partition] : Object.keys(this._ledger[t]);
+                  [partition] : Object.keys(this.#ledger[t]);
             partitions.forEach(p => {
-                const partitionOffsets = this._ledger[t][p];
+                const partitionOffsets = this.#ledger[t][p];
                 count += partitionOffsets.processing.length;
             });
         });
@@ -139,12 +136,10 @@ class OffsetLedger {
     /**
      * Export the ledger in JSON format (useful for debugging)
      *
-     * @return {string} a JSON-serialized representation of the
+     * @return a JSON-serialized representation of the
      * current state of the ledger
      */
-    toString() {
-        return JSON.stringify(this._ledger);
+    toString(): string {
+        return JSON.stringify(this.#ledger);
     }
 }
-
-module.exports = OffsetLedger;