Skip to content

Commit

Permalink
Add archiver-types
Browse files Browse the repository at this point in the history
  • Loading branch information
Paul Varache committed Aug 7, 2018
1 parent e74e2c7 commit 765e37d
Show file tree
Hide file tree
Showing 13 changed files with 1,936 additions and 33 deletions.
1 change: 1 addition & 0 deletions .npmignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
test
19 changes: 18 additions & 1 deletion bin/kart
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,7 @@ function archive(argv) {
packagejson,
name,
version,
type,
metadata;

if (argv.fromRepo) {
Expand All @@ -257,8 +258,18 @@ function archive(argv) {

name = argv.name ? argv.name : name;
version = argv.buildVersion ? argv.buildVersion : version;
type = argv.type ? argv.type : 'tar.gz';

return kart.archive.store(argv.buildDir, name, argv.channel, version, null, argv.arch, metadata)
return kart.archive.store(
argv.buildDir,
name,
argv.channel,
version,
null,
argv.arch,
metadata,
type,
)
.then((b) => {
build = b;

Expand Down Expand Up @@ -405,6 +416,12 @@ let argv = yargs
'release': {
describe: 'release after archiving',
type: 'boolean'
},
'archive-type': {
alias: 't',
describe: 'set the compression type (defaults to tar.gz)',
choices: ['tar.gz', 'zip', 'ditto', 'none'],
default: 'tar.gz'
}
}, (argv) => {
handleCommonOptions(argv).then(() => {
Expand Down
54 changes: 31 additions & 23 deletions lib/archive.js
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
const tarFs = require('tar-fs'),
fs = require('fs'),
zlib = require('zlib'),
const fs = require('fs'),
semver = require('semver'),
config = require('./config'),
Build = require('./data').Build,
s3 = require('./s3-helpers');
s3 = require('./s3-helpers'),
archivers = {
'tar.gz': require('./archivers/tar.gz.js'),
none: require('./archivers/none.js'),
ditto: require('./archivers/ditto.js'),
zip: require('./archivers/zip.js'),
};

function _getNextBuildNumber(projectName, channel, version) {
let opts = {
Expand Down Expand Up @@ -38,10 +42,12 @@ function _getNextBuildNumber(projectName, channel, version) {
* Will be determined automatically when omitted.
* @param {String} arch (optional) Architecture for this build. Defaults to 'all'.
* @param {Object} metadata (optional) Additional info to store with the build.
* @param {string} type (optional) Archive type. tar.gz, zip or none
* @param {String} .revision Git SHA to tie the build with a particular commit.
*/
function storeBuild(buildDir, projectName, channel, version, number, arch, metadata) {
function storeBuild(buildDir, projectName, channel, version, number, arch, metadata, type) {
arch = arch || 'all';
type = type || 'tar.gz';

if (Object.keys(config.remote.projects).indexOf(projectName) < 0) {
return Promise.reject(`Project '${projectName}' not found`);
Expand All @@ -52,45 +58,47 @@ function storeBuild(buildDir, projectName, channel, version, number, arch, metad
}

if (number) {
return _doStoreBuild(buildDir, projectName, channel, version, number, arch, metadata);
return _doStoreBuild(buildDir, projectName, channel, version, number, arch, metadata, type);
} else {
return _getNextBuildNumber(projectName, channel, version)
.then((n) => {
return _doStoreBuild(buildDir, projectName, channel, version, n, arch, metadata);
return _doStoreBuild(buildDir, projectName, channel, version, n, arch, metadata, type);
});
}
}

function _doStoreBuild(buildDir, projectName, channel, version, number, arch, metadata) {
function _doStoreBuild(buildDir, projectName, channel, version, number, arch, metadata, type) {
return new Promise((resolve, reject) => {
fs.lstat(buildDir, (err, stat) => {
if (err) {
return reject(new Error(`Could not find or access build directory: ${buildDir}`));
}

if (!stat.isDirectory()) {
return reject(new Error(`Path provided is not a directory: ${buildDir}`));
const archiver = archivers[type];

if (!archiver) {
return reject(new Error(`Archiver ${type} does not exist`));
}

let gzip = zlib.createGzip(),
stream = tarFs.pack(buildDir),
build = new Build({
project: projectName,
channel: channel,
version: version,
metadata: metadata,
number: number,
arch: arch,
buildDate: new Date()
});
const stream = archiver.archive(buildDir);
const build = new Build({
project: projectName,
channel: channel,
version: version,
metadata: metadata,
number: number,
arch: arch,
ext: archiver.extension(buildDir),
buildDate: new Date()
});

s3.getInstance().upload(
{
Bucket: config.local.rootBucket.name,
Key: build.path,
Body: stream.pipe(gzip),
Body: stream,
ACL: 'public-read',
Metadata: metadata
Metadata: metadata,
},
(err, data) => {
if (err) {
Expand Down
40 changes: 40 additions & 0 deletions lib/archivers/ditto.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
const cp = require('child_process');
const path = require('path');
const fs = require('fs');
const os = require('os');

function createZipWithDitto(basedir, file, target) {
target = target + '.zip';
return new Promise((resolve, reject) => {
let cmd = 'ditto',
args = ['-ck', '--rsrc', '--sequesterRsrc', '--keepParent', file, target],
p;
console.log(`[ZIP] ${cmd} ${args.join(' ')}`);
p = cp.spawn(cmd, args, { cwd: basedir });
p.on('error', (e) => reject(e));
p.stdout.on('data', (d) => {
console.log(`[ZIP] ${d.toString()}`);
});
p.on('exit', (code) => {
if (code != 0) {
throw new Error(`ditto exited with a non-zero code: ${code}`);
}
resolve(target);
});
});
}

module.exports = {
archive(buildDir) {
const baseDir = path.dirname(buildDir);
const name = buildDir.split('/').pop();
const target = path.join(os.tmpdir(), 'kart-ditto-tmp');
return createZipWithDitto(baseDir, name, target)
.then(() => {
return fs.createReadStream(`${target}.zip`);
});
},
extension() {
return 'zip';
},
};
11 changes: 11 additions & 0 deletions lib/archivers/none.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
const fs = require('fs');
const path = require('path');

module.exports = {
archive(buildDir) {
return fs.createReadStream(buildDir);
},
extension(buildDir) {
return path.extname(buildDir).replace(/^\./, '');
},
};
13 changes: 13 additions & 0 deletions lib/archivers/tar.gz.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
const tarFs = require('tar-fs'),
zlib = require('zlib');

module.exports = {
archive(buildDir) {
let gzip = zlib.createGzip(),
stream = tarFs.pack(buildDir);
return stream.pipe(gzip);
},
extension() {
return 'tar.gz';
},
};
18 changes: 18 additions & 0 deletions lib/archivers/zip.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
const archiver = require('archiver');
const path = require('path');

module.exports = {
archive(buildDir) {
const archive = archiver('zip');
archive
.glob(path.normalize(`**/*`), {
cwd: path.normalize(buildDir),
root: path.normalize(buildDir),
})
.finalize();
return archive;
},
extension() {
return 'zip';
},
};
6 changes: 4 additions & 2 deletions lib/data.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,12 @@ class Build {
this.arch = opts.arch;
this.metadata = opts.metadata || {};
this.buildDate = new Date(opts.buildDate);
this.ext = opts.ext || 'tar.gz';
}

static fromListEntry (entry) {
// (-([a-fA-F0-9]{7}))?
let found = entry.Key.match(/([^\/]+)\/([^\/]+)\/([^_]+)_([^_-]+)-(\d+)_([^_]+)\.tar\.gz/);
let found = entry.Key.match(/([^\/]+)\/([^\/]+)\/([^_]+)_([^_-]+)-(\d+)_([^_]+?)\.(.+)$/);

if (!found) {
return null;
Expand All @@ -26,14 +27,15 @@ class Build {
version: found[4],
number: parseInt(found[5]),
arch: found[6],
ext: found[7],
metadata: {},
buildDate: new Date(entry.LastModified)
});
}

get path() {
return `${this.project}/${this.channel}/` +
`${this.project}_${this.buildVersion}_${this.arch}.tar.gz`;
`${this.project}_${this.buildVersion}_${this.arch}.${this.ext}`;
}

get publicUrl() {
Expand Down
7 changes: 5 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
},
"homepage": "https://github.com/KanoComputing/kart#readme",
"dependencies": {
"archiver": "^2.1.1",
"async": "^2.6.0",
"aws-sdk": "^2.111.0",
"colors": "^1.1.2",
Expand All @@ -32,17 +33,19 @@
"tar-fs": "^1.16.0",
"tar-stream": "^1.5.5",
"update-notifier": "^2.3.0",
"yargs": "^8.0.2"
"yargs": "^8.0.2",
"zip-stream": "^1.2.0"
},
"devDependencies": {
"inquirer-test": "^2.0.1",
"istanbul": "^0.4.5",
"mocha": "^4.0.1",
"mock-aws-s3": "^2.6.0",
"should": "^13.2.0",
"strip-ansi": "^4.0.0",
"tmp": "0.0.33"
},
"engines": {
"node": ">=6.x"
"node": ">=6.x"
}
}
82 changes: 82 additions & 0 deletions test/archivers.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
const kart = require('../lib'),
should = require('should'),
testUtil = require('./test-util'),
path = require('path');

describe('kart.archive', function () {
this.timeout(30000);

beforeEach(() => {
return testUtil.setupS3();
});
afterEach(() => {
testUtil.cleanupBuildDirectories();
return testUtil.teardownS3();
});
describe('.store()', () => {
it('with zip type', () => {
let buildDir;

return testUtil.generateBuildDirectory({
fileCount: [10, 20],
}).then((dir) => {
buildDir = dir;
return kart.archive.store(buildDir.path, 'testing', 'sync', '0.5.6', null, 'armv7', null, 'zip');
}).then((archive) => {
archive.should.containEql({
project: 'testing',
channel: 'sync',
version: '0.5.6',
number: 1,
arch: 'armv7',
ext: 'zip',
});
});
});
it('with no type', () => {
let buildDir;

return testUtil.generateBuildDirectory({
fileCount: 1,
}).then((dir) => {
buildDir = dir;
return kart.archive.store(path.join(buildDir.path, buildDir.files[0]), 'testing', 'sync', '0.5.6', null, 'armv7', null, 'none');
}).then((archive) => {
archive.should.containEql({
project: 'testing',
channel: 'sync',
version: '0.5.6',
number: 1,
arch: 'armv7',
ext: 'txt',
});
});
});
if (process.platform !== 'darwin') {
return;
}
it('with ditto', () => {
let buildDir;

return testUtil.generateBuildDirectory({
fileCount: [10, 20],
}).then((dir) => {
buildDir = dir;
return kart.archive.store(buildDir.path, 'testing', 'sync', '0.5.6', null, 'armv7', null, 'ditto');
}).then((archive) => {
archive.should.containEql({
project: 'testing',
channel: 'sync',
version: '0.5.6',
number: 1,
arch: 'armv7',
ext: 'zip',
});
});
});
});
});

setTimeout(() => {

}, 4000000000000000000000000);
Loading

0 comments on commit 765e37d

Please sign in to comment.