Skip to content

Commit

Permalink
fix typesize, fix composit flags, make pretty
Browse files Browse the repository at this point in the history
  • Loading branch information
@fangq
fangq committed Sep 13, 2022
1 parent 552a007 commit 78b1fa9
Show file tree
Hide file tree
Showing 5 changed files with 75 additions and 68 deletions.
2 changes: 1 addition & 1 deletion include/zmatlib.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ extern "C"
* -1: unknown
*/

enum TZipMethod {zmZlib, zmGzip, zmBase64, zmLzip, zmLzma, zmLz4, zmLz4hc, zmBlosc2Blosclz, zmBlosc2Lz4, zmBlosc2Lz4hc, zmBlosc2Zlib, zmBlosc2Zstd, zmUnknown=-1};
enum TZipMethod {zmZlib, zmGzip, zmBase64, zmLzip, zmLzma, zmLz4, zmLz4hc, zmBlosc2Blosclz, zmBlosc2Lz4, zmBlosc2Lz4hc, zmBlosc2Zlib, zmBlosc2Zstd, zmUnknown = -1};

/**
* @brief Main interface to perform compression/decompression
Expand Down
2 changes: 1 addition & 1 deletion src/blosc2/blosc/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ PLATFORM = $(shell uname -s)
DLLFLAG=-fPIC
OMP=-fopenmp

CPPOPT=-g -Wall -O3 -msse2 -DHAVE_ZSTD -DHAVE_ZLIB -DHAVE_LZ4 $(DLLFLAG) #-g -Wall -std=c99 # -DUSE_OS_TIMER
CPPOPT=-g -Wall -Wextra -O3 -msse2 -DHAVE_ZSTD -DHAVE_ZLIB -DHAVE_LZ4 -DNDEBUG $(DLLFLAG) -std=gnu99 #-g -Wall -std=c99 # -DUSE_OS_TIMER

OUTPUTFLAG:=-o
OBJSUFFIX=.o
Expand Down
79 changes: 41 additions & 38 deletions src/zmat.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,56 +65,62 @@ const char* metadata[] = {"type", "size", "byte", "method", "status", "level"};

void mexFunction(int nlhs, mxArray* plhs[], int nrhs, const mxArray* prhs[]) {
TZipMethod zipid = zmZlib;
int iscompress = 1;
const char* zipmethods[] = {
"zlib",
"gzip",
"base64",
"gzip",
"base64",
#if !defined(NO_LZMA)
"lzip",
"lzma",
"lzma",
#endif
#if !defined(NO_LZ4)
"lz4",
"lz4hc",
"lz4hc",
#endif
#if !defined(NO_BLOSC2)
"blosc2blosclz",
"blosc2lz4",
"blosc2lz4hc",
"blosc2zlib",
"blosc2zstd",
"blosc2lz4",
"blosc2lz4hc",
"blosc2zlib",
"blosc2zstd",
#endif
""};
""
};

const TZipMethod zipmethodid[] = {
zmZlib,
zmGzip,
zmBase64,
zmGzip,
zmBase64,
#if !defined(NO_LZMA)
zmLzip,
zmLzma,
zmLzma,
#endif
#if !defined(NO_LZ4)
zmLz4,
zmLz4hc,
zmLz4hc,
#endif
#if !defined(NO_BLOSC2)
zmBlosc2Blosclz,
zmBlosc2Lz4,
zmBlosc2Lz4hc,
zmBlosc2Zlib,
zmBlosc2Zstd,
zmBlosc2Lz4,
zmBlosc2Lz4hc,
zmBlosc2Zlib,
zmBlosc2Zstd,
#endif
zmUnknown};

int nthread = 1; /*nthread, shuffle and typesize are only used by blosc2 compressors*/
int shuffle = 1;
int typesize = 4;
char clevel = 1;
zmUnknown
};

int use4bytedim = 0;

union cflag {
int iscompress;
struct settings {
char clevel;
char nthread;
char shuffle;
char typesize;
} param;
} flags = {0};

/**
* If no input is given for this function, it prints help information and return.
*/
Expand All @@ -139,9 +145,13 @@ void mexFunction(int nlhs, mxArray* plhs[], int nrhs, const mxArray* prhs[]) {
mxDestroyArray(tmp);
}

flags.param.nthread = 1;
flags.param.shuffle = 1;
flags.param.typesize = 4;

if (nrhs >= 2) {
double* val = mxGetPr(prhs[1]);
clevel = val[0];
flags.param.clevel = (int)val[0];
}

if (nrhs >= 3) {
Expand All @@ -155,30 +165,23 @@ void mexFunction(int nlhs, mxArray* plhs[], int nrhs, const mxArray* prhs[]) {
mexErrMsgTxt("the specified compression method is not supported");
}

zipid = zipmethodid[(int)zipid];
zipid = zipmethodid[(int)zipid];
}

if (nrhs >= 4) {
double* val = mxGetPr(prhs[3]);
nthread = val[0];
flags.param.nthread = val[0];
}

if (nrhs >= 5) {
double* val = mxGetPr(prhs[4]);
shuffle = val[0];
flags.param.shuffle = val[0];
}

if (nrhs >= 6) {
double* val = mxGetPr(prhs[5]);
typesize = val[0];
}
printf("iscompress=%X %d %d %d %X\n", clevel, nthread, shuffle, typesize, ((nthread & 0xFF) << 8));
if(clevel) {
iscompress = (clevel | ((nthread & 0xFF) << 8) | ((shuffle & 0xFF) << 16) | ((typesize & 0xFF) << 24));
} else {
iscompress = (clevel | ((nthread & 0xFF) << 8));
flags.param.typesize = val[0];
}
printf("iscompress=%X\n", iscompress);

try {
if (mxIsChar(prhs[0]) || (mxIsNumeric(prhs[0]) && !mxIsComplex(prhs[0])) || mxIsLogical(prhs[0])) {
Expand All @@ -191,7 +194,7 @@ void mexFunction(int nlhs, mxArray* plhs[], int nrhs, const mxArray* prhs[]) {
int errcode = 0;

if (inputsize > 0) {
errcode = zmat_run(inputsize, inputstr, &outputsize, &outputbuf, zipid, &ret, iscompress);
errcode = zmat_run(inputsize, inputstr, &outputsize, &outputbuf, zipid, &ret, flags.iscompress);
}

if (errcode < 0) {
Expand Down Expand Up @@ -265,7 +268,7 @@ void mexFunction(int nlhs, mxArray* plhs[], int nrhs, const mxArray* prhs[]) {
mxSetFieldByNumber(plhs[1], 0, 4, val);

val = mxCreateDoubleMatrix(1, 1, mxREAL);
*mxGetPr(val) = iscompress;
*mxGetPr(val) = flags.param.clevel;
mxSetFieldByNumber(plhs[1], 0, 5, val);
}

Expand Down
57 changes: 30 additions & 27 deletions src/zmatlib.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,34 +152,35 @@ char* zmat_error(int id) {
int zmat_run(const size_t inputsize, unsigned char* inputstr, size_t* outputsize, unsigned char** outputbuf, const int zipid, int* ret, const int iscompress) {
z_stream zs;
size_t buflen[2] = {0};
unsigned int nthread=1, shuffle=1, typesize=4;
char compressflag = iscompress & 0xFF;
unsigned int nthread = 1, shuffle = 1, typesize = 4;
int clevel;
union cflag {
int iscompress;
struct settings {
char clevel;
char nthread;
char shuffle;
char typesize;
} param;
} flags;
*outputbuf = NULL;

flags.iscompress = iscompress;

zs.zalloc = Z_NULL;
zs.zfree = Z_NULL;
zs.opaque = Z_NULL;

if (inputsize == 0) {
return -1;
}
printf("flag=%X\n", iscompress);
if((iscompress & 0xFF00) >> 8) {
nthread = (iscompress & 0xFF00) >> 8;
}
printf("nthread=%d\n", nthread);
if((iscompress & 0xFF0000) >> 16) {
shuffle = (iscompress & 0xFF0000) >> 16;
}
printf("shuffle=%d\n", shuffle);

if((iscompress & 0xFF000000) >> 24) {
typesize = (iscompress & 0xFF000000) >> 24;
}
printf("typesize=%d\n", typesize);
nthread = (flags.param.nthread == 0) ? 1 : flags.param.nthread;
shuffle = (flags.param.shuffle == 0) ? 1 : flags.param.shuffle;
typesize = (flags.param.typesize == 0) ? 4 : flags.param.typesize;
clevel = (flags.param.clevel == 0) ? 0 : flags.param.clevel;

printf("flag=%X\n", compressflag);
if (compressflag) {
if (clevel) {
/**
* perform compression or encoding
*/
Expand All @@ -193,11 +194,11 @@ printf("flag=%X\n", compressflag);
* zlib (.zip) or gzip (.gz) compression
*/
if (zipid == zmZlib) {
if (deflateInit(&zs, (compressflag > 0) ? Z_DEFAULT_COMPRESSION : (-compressflag)) != Z_OK) {
if (deflateInit(&zs, (clevel > 0) ? Z_DEFAULT_COMPRESSION : (-clevel)) != Z_OK) {
return -2;
}
} else {
if (deflateInit2(&zs, (compressflag > 0) ? Z_DEFAULT_COMPRESSION : (-compressflag), Z_DEFLATED, 15 | 16, MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY) != Z_OK) {
if (deflateInit2(&zs, (clevel > 0) ? Z_DEFAULT_COMPRESSION : (-clevel), Z_DEFLATED, 15 | 16, MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY) != Z_OK) {
return -2;
}
}
Expand Down Expand Up @@ -225,7 +226,7 @@ printf("flag=%X\n", compressflag);
* lzma (.lzma) or lzip (.lzip) compression
*/
*ret = simpleCompress((elzma_file_format)(zipid - 3), (unsigned char*)inputstr,
inputsize, outputbuf, outputsize, compressflag);
inputsize, outputbuf, outputsize, clevel);

if (*ret != ELZMA_E_OK) {
return -4;
Expand All @@ -246,7 +247,7 @@ printf("flag=%X\n", compressflag);
if (zipid == zmLz4) {
*outputsize = LZ4_compress_default((const char*)inputstr, (char*)(*outputbuf), inputsize, *outputsize);
} else {
*outputsize = LZ4_compress_HC((const char*)inputstr, (char*)(*outputbuf), inputsize, *outputsize, (compressflag > 0) ? 8 : (-compressflag));
*outputsize = LZ4_compress_HC((const char*)inputstr, (char*)(*outputbuf), inputsize, *outputsize, (clevel > 0) ? 8 : (-clevel));
}

*ret = *outputsize;
Expand All @@ -261,19 +262,21 @@ printf("flag=%X\n", compressflag);
/**
* blosc2 meta-compressor (support various filters and compression codecs)
*/
const char *codecs[] ={"blosclz", "lz4", "lz4hc", "zlib", "zstd"};
if (blosc1_set_compressor(codecs[zipid - zmBlosc2Blosclz]) == -1) {
return -7;
}
const char* codecs[] = {"blosclz", "lz4", "lz4hc", "zlib", "zstd"};

blosc2_set_nthreads(nthread);
if (blosc1_set_compressor(codecs[zipid - zmBlosc2Blosclz]) == -1) {
return -7;
}

blosc2_set_nthreads(nthread);

*outputsize = inputsize + BLOSC2_MAX_OVERHEAD; /* blosc2 guarantees the compression will always succeed at this size */

if (!(*outputbuf = (unsigned char*)malloc(*outputsize))) {
return -5;
}
*ret = blosc1_compress((compressflag > 0) ? 5 : (-compressflag), shuffle, typesize, inputsize, (const void*)inputstr, (void*)(*outputbuf), *outputsize);

*ret = blosc1_compress((clevel > 0) ? 5 : (-clevel), shuffle, typesize, inputsize, (const void*)inputstr, (void*)(*outputbuf), *outputsize);

*outputsize = *ret;

Expand Down
3 changes: 2 additions & 1 deletion zmat.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,8 @@
error('input must be a char, non-complex numeric or logical vector or N-D array');
end

typesize=length(typecast(input(1), 'uint8'));
inputinfo=whos('input');
typesize=inputinfo.bytes/numel(input);

if (ischar(input))
input = uint8(input);
Expand Down

0 comments on commit 78b1fa9

Please sign in to comment.