Skip to content

Commit

Permalink
Merge pull request #358 from sisong/dev
Browse files Browse the repository at this point in the history
patch compatible with another BSDIFF format "ENDSLEY/BSDIFF43"
  • Loading branch information
sisong authored Aug 31, 2023
2 parents fd75602 + 7e86c2e commit 7a799d5
Show file tree
Hide file tree
Showing 12 changed files with 139 additions and 58 deletions.
16 changes: 16 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,27 @@

full changelog at: https://github.com/sisong/HDiffPatch/commits

## [v4.6.7](https://github.com/sisong/HDiffPatch/tree/v4.6.7) - 2023-08-31
### Added
* patch compatible with another BSDIFF format "ENDSLEY/BSDIFF43", https://github.com/mendsley/bsdiff

## [v4.6.6](https://github.com/sisong/HDiffPatch/tree/v4.6.6) - 2023-08-27
### Added
* patch demo SDK for iOS, out lib file libhpatchz.a;

## [v4.6.3](https://github.com/sisong/HDiffPatch/tree/v4.6.3) - 2023-05-19
### Added
* support for block devices on linux; by author [Alexander Zakharov](https://github.com/uglym8);

## [v4.6.0](https://github.com/sisong/HDiffPatch/tree/v4.6.0) - 2023-04-20
### Added
* add libhsync for diff&patch by sync; see demo [hsynz](https://github.com/sisong/hsynz) (like [zsync](http://zsync.moria.org.uk))
* add function create_sync_data(),create_dir_sync_data(),sync_patch(),sync_patch_...(),sync_local_diff(),sync_local_diff_...(),sync_local_patch(),sync_local_patch_...()

## [v4.5.2](https://github.com/sisong/HDiffPatch/tree/v4.5.2) - 2022-12-25
### Fixed
* fix a bug when run dir_diff by muti-thread parallel;

## [v4.5.0](https://github.com/sisong/HDiffPatch/tree/v4.5.0) - 2022-11-23
### Added
* cmdline added option "-VCD[-compressLevel[-dictSize]]", create diffFile compatible with VCDIFF format;
Expand Down
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# [HDiffPatch](https://github.com/sisong/HDiffPatch)
[![release](https://img.shields.io/badge/release-v4.6.6-blue.svg)](https://github.com/sisong/HDiffPatch/releases)
[![release](https://img.shields.io/badge/release-v4.6.7-blue.svg)](https://github.com/sisong/HDiffPatch/releases)
[![license](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/sisong/HDiffPatch/blob/master/LICENSE)
[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-blue.svg)](https://github.com/sisong/HDiffPatch/pulls)
[![+issue Welcome](https://img.shields.io/github/issues-raw/sisong/HDiffPatch?color=green&label=%2Bissue%20welcome)](https://github.com/sisong/HDiffPatch/issues)
Expand Down Expand Up @@ -109,7 +109,7 @@ special options:
create diffFile compatible with bsdiff4, unsupport input directory(folder).
-VCD[-compressLevel[-dictSize]]
create diffFile compatible with VCDIFF, unsupport input directory(folder).
DEFAULT no compress, out format same as $open-vcdiff delta ... or $xdelta3 -S -e -n ...
DEFAULT no compress, out format same as $open-vcdiff ... or $xdelta3 -S -e -n ...
if set compressLevel, out format same as $xdelta3 -S lzma -e -n ...
compress by 7zXZ(xz), compressLevel in {0..9}, DEFAULT level 7;
dictSize can like 4096 or 4k or 4m or 16m etc..., DEFAULT 8m
Expand Down
2 changes: 1 addition & 1 deletion README_cmdline_cn.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
创建一个和bsdiff4兼容的补丁, 不支持参数为文件夹。
-VCD[-compressLevel[-dictSize]]
创建一个标准规范VCDIFF格式的补丁, 不支持参数为文件夹。
默认输出补丁不带压缩, 格式和 $open-vcdiff delta ... 或 $xdelta3 -S -e -n ... 命令输出的补丁格式兼容;
默认输出补丁不带压缩, 格式和 $open-vcdiff ... 或 $xdelta3 -S -e -n ... 命令输出的补丁格式兼容;
如果设置了压缩级别compressLevel, 那输出格式和 $xdelta3 -S lzma -e -n ...命令输出的补丁格式兼容;
压缩输出时补丁文件使用7zXZ(xz)算法压缩, compressLevel可以选择0到9, 默认级别7;
压缩字典大小dictSize可以设置为 4096, 4k, 4m, 16m等, 默认为8m
Expand Down
73 changes: 51 additions & 22 deletions bsdiff_wrapper/bspatch_wrapper.c
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,11 @@ static const char* kBsDiffVersionType = "BSDIFF40";
#define kBsDiffVersionTypeLen 8 // ==strlen(kBsDiffVersionType);
#define kBsDiffHeadLen (kBsDiffVersionTypeLen+3*8)

static const char* kEsBsDiffVersionType = "ENDSLEY/BSDIFF43";
#define kEsBsDiffVersionTypeLen 16 // ==strlen(kEsBsDiffVersionType);
#define kEsBsDiffHeadLen (kEsBsDiffVersionTypeLen+8)


static hpatch_inline hpatch_uint32_t _readUInt32(const unsigned char* buf){
return buf[0] | (((hpatch_uint32_t)buf[1])<<8) |
(((hpatch_uint32_t)buf[2])<<16) | (((hpatch_uint32_t)buf[3])<<24) ;
Expand All @@ -65,18 +70,31 @@ hpatch_BOOL getBsDiffInfo(hpatch_BsDiffInfo* out_diffinfo,const hpatch_TStreamIn
unsigned char* buf=&_buf[0];
if (diffStream->streamSize<kBsDiffHeadLen)
return _hpatch_FALSE;
if (!diffStream->read(diffStream,0,buf,buf+kBsDiffHeadLen))
if (!diffStream->read(diffStream,0,buf,buf+kBsDiffHeadLen)) // must bz2 compressed size>=8
return _hpatch_FALSE;
if (0!=memcmp(buf,kBsDiffVersionType,kBsDiffVersionTypeLen))
if (0==memcmp(buf,kBsDiffVersionType,kBsDiffVersionTypeLen)){
out_diffinfo->isEsBsd=hpatch_FALSE;
out_diffinfo->headSize=kBsDiffHeadLen;
}else if (0==memcmp(buf,kEsBsDiffVersionType,kEsBsDiffVersionTypeLen)){
out_diffinfo->isEsBsd=hpatch_TRUE;
out_diffinfo->headSize=kEsBsDiffHeadLen;
}else{
return _hpatch_FALSE;
buf+=kBsDiffVersionTypeLen;
out_diffinfo->headSize=kBsDiffHeadLen;
out_diffinfo->ctrlDataSize=readUInt64(buf);
out_diffinfo->subDataSize =readUInt64(buf+8);
out_diffinfo->newDataSize =readUInt64(buf+8*2);
}
if (out_diffinfo->isEsBsd){
buf+=kEsBsDiffVersionTypeLen;
out_diffinfo->ctrlDataSize=0;
out_diffinfo->subDataSize =0;
out_diffinfo->newDataSize =readUInt64(buf);
}else{
buf+=kBsDiffVersionTypeLen;
out_diffinfo->ctrlDataSize=readUInt64(buf);
out_diffinfo->subDataSize =readUInt64(buf+8);
out_diffinfo->newDataSize =readUInt64(buf+8*2);
}
return (out_diffinfo->ctrlDataSize<diffStream->streamSize)&&
(out_diffinfo->subDataSize<diffStream->streamSize)&&
(kBsDiffHeadLen+out_diffinfo->ctrlDataSize+out_diffinfo->subDataSize<diffStream->streamSize);
(out_diffinfo->headSize+out_diffinfo->ctrlDataSize+out_diffinfo->subDataSize<diffStream->streamSize);
}

hpatch_BOOL getBsDiffInfo_mem(hpatch_BsDiffInfo* out_diffinfo,const unsigned char* diffData,const unsigned char* diffData_end){
Expand Down Expand Up @@ -175,6 +193,8 @@ hpatch_BOOL bspatch_with_cache(const hpatch_TStreamOutput* out_newData,
hpatch_BsDiffInfo diffInfo;
TStreamCacheClip ctrlClip;
TStreamCacheClip subClip;
TStreamCacheClip* _ctrlClip=&ctrlClip;
TStreamCacheClip* _subClip=&subClip;
TStreamCacheClip newDataDiffClip;
_TDecompressInputStream decompressers[3];
hpatch_size_t i;
Expand Down Expand Up @@ -206,29 +226,38 @@ hpatch_BOOL bspatch_with_cache(const hpatch_TStreamOutput* out_newData,
temp_cache+=cacheSize;
cacheSize=(temp_cache_end-temp_cache);
}
cacheSize=cacheSize/_kCacheBsDecCount;
cacheSize=cacheSize/(diffInfo.isEsBsd?_kCacheBsDecCount-2:_kCacheBsDecCount);
if (cacheSize<8) return _hpatch_FALSE;

diffPos0=diffInfo.headSize;
if (!getStreamClip(&ctrlClip,&decompressers[0],
_kUnknowMaxSize,diffInfo.ctrlDataSize,compressedDiff,&diffPos0,
decompressPlugin,temp_cache,cacheSize)) _clear_return(_hpatch_FALSE);
temp_cache+=cacheSize;
if (!getStreamClip(&subClip,&decompressers[1],
_kUnknowMaxSize,diffInfo.subDataSize,compressedDiff,&diffPos0,
decompressPlugin,temp_cache,cacheSize)) _clear_return(_hpatch_FALSE);
temp_cache+=cacheSize;
if (!getStreamClip(&newDataDiffClip,&decompressers[2],
_kUnknowMaxSize,compressedDiff->streamSize-diffPos0,compressedDiff,&diffPos0,
decompressPlugin,temp_cache,cacheSize)) _clear_return(_hpatch_FALSE);
temp_cache+=cacheSize;
if (diffInfo.isEsBsd){
if (!getStreamClip(&newDataDiffClip,&decompressers[0],
_kUnknowMaxSize,compressedDiff->streamSize-diffPos0,compressedDiff,&diffPos0,
decompressPlugin,temp_cache,cacheSize)) _clear_return(_hpatch_FALSE);
temp_cache+=cacheSize;
_ctrlClip=&newDataDiffClip;
_subClip=&newDataDiffClip;
}else{
if (!getStreamClip(&ctrlClip,&decompressers[0],
_kUnknowMaxSize,diffInfo.ctrlDataSize,compressedDiff,&diffPos0,
decompressPlugin,temp_cache,cacheSize)) _clear_return(_hpatch_FALSE);
temp_cache+=cacheSize;
if (!getStreamClip(&subClip,&decompressers[1],
_kUnknowMaxSize,diffInfo.subDataSize,compressedDiff,&diffPos0,
decompressPlugin,temp_cache,cacheSize)) _clear_return(_hpatch_FALSE);
temp_cache+=cacheSize;
if (!getStreamClip(&newDataDiffClip,&decompressers[2],
_kUnknowMaxSize,compressedDiff->streamSize-diffPos0,compressedDiff,&diffPos0,
decompressPlugin,temp_cache,cacheSize)) _clear_return(_hpatch_FALSE);
temp_cache+=cacheSize;
}
assert(diffPos0==compressedDiff->streamSize);

{
_TOutStreamCache outCache;
_TOutStreamCache_init(&outCache,out_newData,temp_cache,cacheSize);
temp_cache+=cacheSize;
result=bspatchByClip(&outCache,oldData,&ctrlClip,&subClip,&newDataDiffClip,
result=bspatchByClip(&outCache,oldData,_ctrlClip,_subClip,&newDataDiffClip,
temp_cache,cacheSize);
}

Expand Down
1 change: 1 addition & 0 deletions bsdiff_wrapper/bspatch_wrapper.h
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ typedef struct hpatch_BsDiffInfo{
hpatch_StreamPos_t ctrlDataSize;
hpatch_StreamPos_t subDataSize;
hpatch_StreamPos_t newDataSize;
hpatch_BOOL isEsBsd;
} hpatch_BsDiffInfo;

hpatch_BOOL getIsBsDiff(const hpatch_TStreamInput* diffData);
Expand Down
35 changes: 28 additions & 7 deletions builds/android_ndk_jni_mk/Android.mk
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,12 @@ LOCAL_MODULE := hpatchz

# args
ZSTD := 1
LZMA := 1
BROTLI:= 0
VCD := 0
# if open BSD,must open BZIP2
BSD := 0
VCD := 0
BZIP2 := 0
LZMA := 1

ifeq ($(BZIP2),0)
Bz2_Files :=
Expand All @@ -32,9 +33,9 @@ else
LZMA_PATH := $(LOCAL_PATH)/../../../lzma/C/
Lzma_Files := $(LZMA_PATH)/LzmaDec.c \
$(LZMA_PATH)/Lzma2Dec.c
ifeq ($(TARGET_ARCH_ABI),arm64-v8a)
Lzma_Files += $(LZMA_PATH)/../Asm/arm64/LzmaDecOpt.S
endif
ifeq ($(TARGET_ARCH_ABI),arm64-v8a)
Lzma_Files += $(LZMA_PATH)/../Asm/arm64/LzmaDecOpt.S
endif
ifeq ($(VCD),0)
else
Lzma_Files+=$(LZMA_PATH)/7zCrc.c \
Expand Down Expand Up @@ -70,6 +71,22 @@ else
$(ZSTD_PATH)/decompress/zstd_decompress_block.c
endif

ifeq ($(BROTLI),0)
Brotli_Files:=
else
BROTLI_PATH := $(LOCAL_PATH)/../../../brotli/c/
Brotli_Files:=$(BROTLI_PATH)/common/constants.c \
$(BROTLI_PATH)/common/context.c \
$(BROTLI_PATH)/common/dictionary.c \
$(BROTLI_PATH)/common/platform.c \
$(BROTLI_PATH)/common/shared_dictionary.c \
$(BROTLI_PATH)/common/transform.c \
$(BROTLI_PATH)/dec/bit_reader.c \
$(BROTLI_PATH)/dec/decode.c \
$(BROTLI_PATH)/dec/huffman.c \
$(BROTLI_PATH)/dec/state.c
endif

HDP_PATH := $(LOCAL_PATH)/../../
Hdp_Files := $(HDP_PATH)/file_for_patch.c \
$(HDP_PATH)/libHDiffPatch/HPatch/patch.c
Expand All @@ -79,8 +96,8 @@ else
endif
ifeq ($(VCD),0)
else
Hdp_Files += $(HDP_PATH)/vcdiff_wrapper/vcpatch_wrapper.c
Hdp_Files += $(HDP_PATH)/libHDiffPatch/HDiff/private_diff/limit_mem_diff/adler_roll.c
Hdp_Files += $(HDP_PATH)/vcdiff_wrapper/vcpatch_wrapper.c \
$(HDP_PATH)/libHDiffPatch/HDiff/private_diff/limit_mem_diff/adler_roll.c
endif

Src_Files := $(LOCAL_PATH)/hpatch_jni.c \
Expand Down Expand Up @@ -120,6 +137,10 @@ else
-DDYNAMIC_BMI2=0 -DZSTD_LEGACY_SUPPORT=0 -DZSTD_LIB_DEPRECATED=0 -DHUF_FORCE_DECOMPRESS_X1=1 \
-DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT=1 -DZSTD_NO_INLINE=1 -DZSTD_STRIP_ERROR_STRINGS=1
endif
ifeq ($(BROTLI),0)
else
DEF_FLAGS += -D_CompressPlugin_brotli -I$(BROTLI_PATH)/include
endif

LOCAL_SRC_FILES := $(Src_Files) $(Bz2_Files) $(Lzma_Files) $(Zstd_Files) $(Hdp_Files)
LOCAL_LDLIBS := -llog -lz
Expand Down
4 changes: 2 additions & 2 deletions builds/xcode/hpatchz.xcodeproj/project.pbxproj
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,7 @@
CLANG_WARN_SUSPICIOUS_MOVE = YES;
CLANG_WARN_UNREACHABLE_CODE = YES;
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
CODE_SIGN_IDENTITY = "-";
ENABLE_STRICT_OBJC_MSGSEND = YES;
ENABLE_TESTABILITY = YES;
GCC_C_LANGUAGE_STANDARD = "compiler-default";
Expand Down Expand Up @@ -328,6 +329,7 @@
CLANG_WARN_SUSPICIOUS_MOVE = YES;
CLANG_WARN_UNREACHABLE_CODE = YES;
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
CODE_SIGN_IDENTITY = "-";
DEAD_CODE_STRIPPING = YES;
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
DEPLOYMENT_POSTPROCESSING = YES;
Expand Down Expand Up @@ -358,7 +360,6 @@
isa = XCBuildConfiguration;
buildSettings = {
CLANG_ENABLE_OBJC_WEAK = YES;
CODE_SIGN_IDENTITY = "-";
CODE_SIGN_STYLE = Automatic;
DEVELOPMENT_TEAM = "";
MACOSX_DEPLOYMENT_TARGET = 10.9;
Expand All @@ -371,7 +372,6 @@
isa = XCBuildConfiguration;
buildSettings = {
CLANG_ENABLE_OBJC_WEAK = YES;
CODE_SIGN_IDENTITY = "-";
CODE_SIGN_STYLE = Automatic;
DEVELOPMENT_TEAM = "";
MACOSX_DEPLOYMENT_TARGET = 10.9;
Expand Down
4 changes: 2 additions & 2 deletions hdiffz.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ static void printUsage(){
# ifdef _CompressPlugin_7zXZ
" -VCD[-compressLevel[-dictSize]]\n"
" create diffFile compatible with VCDIFF, unsupport input directory(folder).\n"
" DEFAULT no compress, out format same as $open-vcdiff delta ... or $xdelta3 -S -e -n ...\n"
" DEFAULT no compress, out format same as $open-vcdiff ... or $xdelta3 -S -e -n ...\n"
" if set compressLevel, out format same as $xdelta3 -S lzma -e -n ...\n"
" compress by 7zXZ(xz), compressLevel in {0..9}, DEFAULT level 7;\n"
" dictSize can like 4096 or 4k or 4m or 16m etc..., DEFAULT 8m\n"
Expand All @@ -199,7 +199,7 @@ static void printUsage(){
# else
" -VCD\n"
" create diffFile compatible with VCDIFF, unsupport input directory(folder).\n"
" out format same as $open-vcdiff delta ... or $xdelta3 -S -e -n ...\n"
" out format same as $open-vcdiff ... or $xdelta3 -S -e -n ...\n"
# endif
" NOTE: out diffFile used large source window size!\n"
#endif
Expand Down
2 changes: 1 addition & 1 deletion hpatchz.c
Original file line number Diff line number Diff line change
Expand Up @@ -987,7 +987,7 @@ int hpatch(const char* oldFileName,const char* diffFileName,const char* outNewFi
*decompressPlugin=_bz2DecompressPlugin_unsz;
decompressPlugin->decError=hpatch_dec_ok;
memcpy(diffInfo.compressType,bsCompressType,strlen(bsCompressType)+1);
diffInfo.compressedCount=3;
diffInfo.compressedCount=bsdiffInfo.isEsBsd?1:3;
diffInfo.newDataSize=bsdiffInfo.newDataSize;
diffInfo.oldDataSize=poldData->streamSize; //not saved oldDataSize
isBsDiff=hpatch_TRUE;
Expand Down
2 changes: 1 addition & 1 deletion libHDiffPatch/HPatch/patch_types.h
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ extern "C" {

#define HDIFFPATCH_VERSION_MAJOR 4
#define HDIFFPATCH_VERSION_MINOR 6
#define HDIFFPATCH_VERSION_RELEASE 6
#define HDIFFPATCH_VERSION_RELEASE 7

#define _HDIFFPATCH_VERSION HDIFFPATCH_VERSION_MAJOR.HDIFFPATCH_VERSION_MINOR.HDIFFPATCH_VERSION_RELEASE
#define _HDIFFPATCH_QUOTE(str) #str
Expand Down
32 changes: 21 additions & 11 deletions vcdiff_wrapper/vcdiff_wrapper.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,20 @@
#include "../libHDiffPatch/HPatch/patch.h"
#include <stdexcept> //std::runtime_error
#include <algorithm> //std::inplace_merge
#include <string>
#define _check(value,info) { if (!(value)) { throw std::runtime_error(info); } }
static const hpatch_byte kVcDiffType[3]={('V'|(1<<7)),('C'|(1<<7)),('D'|(1<<7))};
#define kVcDiffVersion 0
static const char* kHDiffzAppHead_a="$hdiffz-VCDIFF&7zXZ#a";

static const char* kHDiffzAppHead="$hdiffz-VCDIFF&";
static const char* kHDiffzAppHead_version="#a";

static std::string getHDiffzAppHead(const vcdiff_TCompress* compressPlugin){
std::string str(kHDiffzAppHead);
if (compressPlugin&&compressPlugin->compress)
str+=compressPlugin->compress->compressType();
return str+kHDiffzAppHead_version;
}

namespace hdiff_private{

Expand Down Expand Up @@ -370,18 +380,18 @@ static void serialize_vcdiff(const hpatch_TStreamInput* newData,const hpatch_TSt
pushBack(buf,kVcDiffType,sizeof(kVcDiffType));
buf.push_back(kVcDiffVersion);
const bool isHaveCompresser=(compressPlugin!=0)&&(compressPlugin->compress_type!=kVcDiff_compressorID_no);
const bool isHDiffzAppHead_a=isHaveCompresser;
hpatch_byte Hdr_Indicator = ((isHaveCompresser?1:0)<<0) // VCD_DECOMPRESS
| (0<<1) // VCD_CODETABLE, no custom code table
| ((isHDiffzAppHead_a?1:0)<<2); // VCD_APPHEADER
if (!isHaveCompresser)
compressPlugin=0;
hpatch_byte Hdr_Indicator = ((isHaveCompresser?1:0)<<0) // VCD_DECOMPRESS
| (0<<1) // VCD_CODETABLE, no custom code table
| (1<<2); // VCD_APPHEADER
buf.push_back(Hdr_Indicator);
if (isHaveCompresser)
if (isHaveCompresser)//VCD_DECOMPRESS
buf.push_back(compressPlugin->compress_type);
else
compressPlugin=0;
if (isHDiffzAppHead_a){
packUInt(buf,strlen(kHDiffzAppHead_a));
pushCStr(buf,kHDiffzAppHead_a);
{ //VCD_APPHEADER // always add HDiffzAppHead tag to out_diff
const std::string HDiffzAppHead=getHDiffzAppHead(compressPlugin);
packUInt(buf,HDiffzAppHead.size());
pushCStr(buf,HDiffzAppHead.c_str());
}
_flushBuf(outDiff,buf);
}
Expand Down
Loading

0 comments on commit 7a799d5

Please sign in to comment.