From b2f281acb121336bc1850edd6790458a4237d7d6 Mon Sep 17 00:00:00 2001 From: Radek Pazdera Date: Fri, 22 Dec 2017 16:52:41 +0000 Subject: [PATCH] WIP: kart test suite --- lib/archive.js | 36 +++---- lib/config.js | 7 +- lib/deploy-methods/s3.js | 26 +++-- lib/index.js | 11 ++- lib/s3-instance.js | 13 +++ package.json | 8 +- test/archive-test.js | 121 +++++++++++++++++++++++ test/playground.js | 34 ------- test/resources/kano-releases-config.json | 23 ----- 9 files changed, 178 insertions(+), 101 deletions(-) create mode 100644 lib/s3-instance.js create mode 100644 test/archive-test.js delete mode 100644 test/playground.js delete mode 100644 test/resources/kano-releases-config.json diff --git a/lib/archive.js b/lib/archive.js index 3379061..9fd8933 100644 --- a/lib/archive.js +++ b/lib/archive.js @@ -1,9 +1,8 @@ -const aws = require('aws-sdk'), - TarGz = require('tar.gz'), +const TarGz = require('tar.gz'), semver = require('semver'), config = require('./config'), - Build = require('./data').Build; - + Build = require('./data').Build, + s3 = require('./s3-instance'); function _getNextBuildNumber(projectName, channel, version) { let opts = { @@ -16,7 +15,7 @@ function _getNextBuildNumber(projectName, channel, version) { }, limit: 1 }; - + return list(projectName, channel, opts).then((res) => { if (res.length > 0) { return res[0].number + 1; @@ -28,7 +27,7 @@ function _getNextBuildNumber(projectName, channel, version) { /* * Compress and upload a build to the apropriate place in the archive. - * + * * @param {String} buildDir Path to a directory to be archived (top level dir won't be included). * @param {String} projectName Name of the project being archived (e.g., kano-code). * @param {String} channel Target channel (e.g., staging). @@ -70,7 +69,6 @@ function _doStoreBuild(buildDir, projectName, channel, version, number, arch, me buildDate: new Date() }); - let s3 = new aws.S3(); s3.upload( { Bucket: config.local.rootBucket.name, @@ -92,9 +90,9 @@ function _doStoreBuild(buildDir, projectName, channel, version, number, arch, me /* * Remove a build from the archive by path or build object. - * + * * @param {String, Object} build Either a direct path to build or a build object. - * + * */ function removeBuild(build) { if (typeof build === 'string') { @@ -102,7 +100,6 @@ function removeBuild(build) { } return new Promise((resolve, reject) => { - let s3 = new aws.S3(); s3.deleteObject( { Bucket: config.local.rootBucket.name, @@ -146,13 +143,13 @@ function _getComparator(opts) { if (opts.sort) { let keys, order = opts.sort.order || 1; - + if (!Array.isArray(opts.sort.key)) { keys = [opts.sort.key]; } else { keys = opts.sort.key.slice(0); } - + while (res === 0 && keys.length) { let key = keys.shift(); @@ -175,36 +172,35 @@ function _getComparator(opts) { } } } - + return res; }; } /* * List builds of a project in a channel. - * + * * @param {String} project The name of the project (e.g., kano-code). * @param {String} channel The name of the channel (e.g., staging). - * + * * @param {Object} opts (optional) Modify the listing results. * @param {Object} opts.filter Filtering options (key value pairs). * Filtering based on metadata not supported at the moment. - * + * * @param {Object} opts.sort Sorting options. * @param {Sring, Array} opts.sort.key One or more keys to sort by in order of priority * @param {Number} opts.sort.order 1 for ascending, -1 for descending. - * + * * @param {Number} opts.limit Limit the list to N entries. * */ function list(project, channel, opts) { opts = Object.assign({}, opts); - + return new Promise((resolve, reject) => { let folder = `${project}/${channel}/`, results; - - let s3 = new aws.S3(); + s3.listObjects( { Bucket: config.local.rootBucket.name, diff --git a/lib/config.js b/lib/config.js index 809c8df..1df4b53 100644 --- a/lib/config.js +++ b/lib/config.js @@ -1,6 +1,6 @@ -const aws = require('aws-sdk'), - fs = require('fs'), - os = require('os'); +const fs = require('fs'), + os = require('os') + s3 = require('./s3-instance'); let configured = false, localConfig = {}, @@ -40,7 +40,6 @@ function configGuard(func) { function _downloadRemoteConfig(root) { return new Promise((resolve, reject) => { - let s3 = new aws.S3(); s3.getObject( { Bucket: root.name, diff --git a/lib/deploy-methods/s3.js b/lib/deploy-methods/s3.js index 2f591d8..d6ce28b 100644 --- a/lib/deploy-methods/s3.js +++ b/lib/deploy-methods/s3.js @@ -1,15 +1,13 @@ -const aws = require('aws-sdk'), - config = require('../config'), +const config = require('../config'), Release = require('../data').Release, mime = require('mime-types'), gunzip = require('gunzip-maybe'), - tarStream = require('tar-stream'); - + tarStream = require('tar-stream'), + s3 = require('../s3-instance'); function _doRelease(build, target, algorithm) { return new Promise((resolve, reject) => { - let s3 = new aws.S3(), - bucketMap = {}, + let bucketMap = {}, extract = tarStream.extract(), dataStream = s3.getObject({ Bucket: config.local.rootBucket.name, @@ -20,7 +18,6 @@ function _doRelease(build, target, algorithm) { reject('Download failed: ' + error); }); - _listAllObjects({ Bucket: target }).then((entries) => { @@ -38,6 +35,7 @@ function _doRelease(build, target, algorithm) { let entry = bucketMap[header.name]; if (entry) { + entry.processed = true; if (entry.Size === header.size && entry.LastModified === header.mtime) { console.log('Skipping ' + header.name); stream.resume(); @@ -95,11 +93,12 @@ function _doRelease(build, target, algorithm) { function _listAllObjects(params, out = []) { return new Promise((resolve, reject) => { - let s3 = new aws.S3(); + s3.listObjectsV2(params, (err, data) => { + if (err) { + return reject(err); + } - s3.listObjectsV2(params).promise() - .then(data => { - out.push(...data.Contents); + out.push(...data.Contents); if (data.IsTruncated) { let newParams = Object.assign(params, {ContinuationToken: data.NextContinuationToken}); @@ -107,13 +106,12 @@ function _listAllObjects(params, out = []) { } else { resolve(out); } - }).catch(reject); + }) }); } function _uploadKartFile(release, target) { return new Promise((resolve, reject) => { - let s3 = new aws.S3(); s3.upload( { Bucket: target, @@ -133,7 +131,6 @@ function _uploadKartFile(release, target) { function _downloadKartFile(project, channel) { return new Promise((resolve, reject) => { - let s3 = new aws.S3(); s3.getObject( { Bucket: config.remote.projects[project].channels[channel].deploy.bucket, @@ -158,7 +155,6 @@ function _deleteEntries(bucketName, entries) { let p = entries.map((entry) => { return new Promise((resolve, reject) => { if (!entry.Key.match(/\/$/)) { - let s3 = new aws.S3(); s3.deleteObject( { Bucket: bucketName, diff --git a/lib/index.js b/lib/index.js index 31c4524..0cba62b 100644 --- a/lib/index.js +++ b/lib/index.js @@ -1,10 +1,10 @@ -const aws = require('aws-sdk'), - semver = require('semver'), +const semver = require('semver'), config = require('./config'), Build = require('./data').Build, Release = require('./data').Release, deployMethods = require('./deploy-methods'), - archive = require('./archive'); + archive = require('./archive'), + s3 = require('./s3-instance'); function getProjects() { return Promise.resolve(config.remote.projects); @@ -49,6 +49,7 @@ function getMOTD() { return Promise.resolve(motd); } + module.exports = { configure: config.configure, archive: { @@ -59,5 +60,7 @@ module.exports = { getProjects: config.guard(getProjects), getMOTD: config.guard(getMOTD), release: config.guard(release), - status: config.guard(status) + status: config.guard(status), + + __mockS3API: s3.__mockS3API }; \ No newline at end of file diff --git a/lib/s3-instance.js b/lib/s3-instance.js new file mode 100644 index 0000000..ba2e4f9 --- /dev/null +++ b/lib/s3-instance.js @@ -0,0 +1,13 @@ +const aws = require('aws-sdk'); + +/* Keep a global reference se we can + override it with a mock for testing. */ +let s3 = new aws.S3(); + +function __mockS3API(mock) { + s3 = mock; + s3.__mockS3API = __mockS3API; +} + +s3.__mockS3API = __mockS3API; +module.exports = s3; \ No newline at end of file diff --git a/package.json b/package.json index 7db8aaf..21cdb72 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ }, "homepage": "https://github.com/KanoComputing/kart#readme", "dependencies": { + "async": "^2.6.0", "aws-sdk": "^2.111.0", "colors": "^1.1.2", "git-rev-sync": "^1.9.1", @@ -27,9 +28,14 @@ "inquirer": "^3.2.3", "mime-types": "^2.1.17", "semver": "^5.4.1", - "tar.gz": "^1.0.7", "tar-stream": "^1.5.5", + "tar.gz": "^1.0.7", "update-notifier": "^2.3.0", "yargs": "^8.0.2" + }, + "devDependencies": { + "mocha": "^4.0.1", + "mock-aws-s3": "^2.6.0", + "tmp": "0.0.33" } } diff --git a/test/archive-test.js b/test/archive-test.js new file mode 100644 index 0000000..03910db --- /dev/null +++ b/test/archive-test.js @@ -0,0 +1,121 @@ +const kart = require('../lib'), + fs = require('fs'), + assert = require('assert'), + tmp = require('tmp'), + async = require('async'), + AWSMock = require('mock-aws-s3'), + ROOT_BUCKET = 'releases-root-testing', + CONFIG_NAME = 'kart-projects.json'; + +let tmpDir = null; + +function generateRandomString(length) { + let result = '', + alphabet = 'abcdefghijklmnopqrstuvwxyzABCDEFGHJKLMNOPQRSTUVWXYZ0123456789 '; + + for (let i = 0; i < length; i++) { + result += alphabet[Math.floor((Math.random() * alphabet.length))]; + } + + return result; +} + +function generateRandomProject() { + return new Promise((resolve, reject) => { + tmp.dir({unsafeCleanup: true}, (err, path, cleanupCallback) => { + if (err) { + return reject(err); + } + + let fileCount= Math.floor((Math.random() * 10) + 1), + fileNames = []; + async.times(fileCount, (i, next) => { + const fileLength = Math.floor((Math.random() * 1024 * 1024) + 2 * 1024), + fileName = `file-${i}.txt`; + + fileNames.push(fileName); + fs.writeFile(`${path}/${fileName}`, generateRandomString(fileLength), next); + }, (err) => { + if (err) { + return reject(err); + } + + resolve({ + path: path, + cleanup: cleanupCallback, + files: fileNames + }); + }); + }); + }); +} + +// Verify upload +function assertUpload () { + +} + +function assetDeploy () { + +} + +describe('kart', function () { + this.timeout(10000); + + before((done) => { + tmpDir = tmp.dirSync({unsafeCleanup: true}); + AWSMock.config.basePath = tmpDir.name; + + let s3 = AWSMock.S3(); + + s3.upload({ + Bucket: ROOT_BUCKET, + Key: CONFIG_NAME, + Body: JSON.stringify({ + projects: { + "testing": { + github: "KanoComputing/testing", + channels: { + sync: { + method: "s3", + bucket: "testing-sync", + algorithm: "sync" + }, + clear: { + method: "s3", + bucket: "testing-clear", + algorithm: "clear" + } + } + } + } + }) + }, (err, data) => { + kart.__mockS3API(s3); + + return kart.configure({ + rootBucket: { + name: ROOT_BUCKET, + config: CONFIG_NAME + } + }).then(() => { + done(); + }); + }); + }); + describe('.archive()', () => { + it('my test', () => { + return generateRandomProject().then((project) => { + console.log(project); + return kart.archive.store(project.path, 'testing', 'testing-sync', '0.1.2', null, null, {revision: '1234567'}); + }).then((build) => { + console.log('lol', build); + }).catch((err) => { + console.log(err); + }); + }); + }); + after(() => { + tmpDir.removeCallback(); + }); +}); diff --git a/test/playground.js b/test/playground.js deleted file mode 100644 index 6784991..0000000 --- a/test/playground.js +++ /dev/null @@ -1,34 +0,0 @@ -const kart = require('../lib'); - -kart.configure({ - // awsKey: process.env.AWS_KEY, - // awsSecret: process.env.AWS_SECRET, - // rootBucket: { - // name: 'releases-root-testing', - // config: 'kano-releases-config.json' - // } -}).then(() => { - - // kart.archive.list('kano-code', 'staging', {sort: {key: ['version', 'build'], order: 1}}) - // .then((data) => { - // console.log(data); - // }).catch((err) => { - // console.log(err); - // }); - - kart.archive.store('./test/resources', 'kano-code', 'staging', '1.10.0', null, null, {revision: '1234567'}) - .then((build) => { - console.log(build); - }) - .catch((err) => { - console.log(err); - }); - - // kart.downloadKartFile('kano-code', 'staging') - // .then((data) => { - // console.log(data); - // }); - -}).catch((err) => { - console.log(err); -}); diff --git a/test/resources/kano-releases-config.json b/test/resources/kano-releases-config.json deleted file mode 100644 index 7036dc8..0000000 --- a/test/resources/kano-releases-config.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "projects": { - "kano-code": { - "compression": "tgz", - "channels": { - "staging": { - "deploy": { - "method": "s3", - "bucket": "releases-target-testing" - }, - "url": "https://apps-staging.kano.me", - }, - "production": { - "deploy": { - "method": "s3", - "bucket": "releases-target-testing" - }, - "url": "https://apps.kano.me" - } - } - } - } -} \ No newline at end of file