Merge pull request #358 from sisong/dev

patch compatible with another BSDIFF format "ENDSLEY/BSDIFF43"

CHANGELOG.md

@@ -2,11 +2,27 @@
 
 full changelog at: https://github.com/sisong/HDiffPatch/commits   
 
+## [v4.6.7](https://github.com/sisong/HDiffPatch/tree/v4.6.7) - 2023-08-31
+### Added
+* patch compatible with another BSDIFF format "ENDSLEY/BSDIFF43", https://github.com/mendsley/bsdiff
+
+## [v4.6.6](https://github.com/sisong/HDiffPatch/tree/v4.6.6) - 2023-08-27
+### Added
+* patch demo SDK for iOS, out lib file libhpatchz.a;
+
+## [v4.6.3](https://github.com/sisong/HDiffPatch/tree/v4.6.3) - 2023-05-19
+### Added
+* support for block devices on linux; by author [Alexander Zakharov](https://github.com/uglym8);
+
 ## [v4.6.0](https://github.com/sisong/HDiffPatch/tree/v4.6.0) - 2023-04-20
 ### Added
 * add libhsync for diff&patch by sync; see demo [hsynz](https://github.com/sisong/hsynz) (like [zsync](http://zsync.moria.org.uk))
 * add function create_sync_data(),create_dir_sync_data(),sync_patch(),sync_patch_...(),sync_local_diff(),sync_local_diff_...(),sync_local_patch(),sync_local_patch_...()
 
+## [v4.5.2](https://github.com/sisong/HDiffPatch/tree/v4.5.2) - 2022-12-25
+### Fixed
+* fix a bug when run dir_diff by muti-thread parallel;
+
 ## [v4.5.0](https://github.com/sisong/HDiffPatch/tree/v4.5.0) - 2022-11-23
 ### Added
 * cmdline added option "-VCD[-compressLevel[-dictSize]]", create diffFile compatible with VCDIFF format;

README.md

@@ -1,5 +1,5 @@
 # [HDiffPatch](https://github.com/sisong/HDiffPatch)
-[![release](https://img.shields.io/badge/release-v4.6.6-blue.svg)](https://github.com/sisong/HDiffPatch/releases) 
+[![release](https://img.shields.io/badge/release-v4.6.7-blue.svg)](https://github.com/sisong/HDiffPatch/releases) 
 [![license](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/sisong/HDiffPatch/blob/master/LICENSE) 
 [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-blue.svg)](https://github.com/sisong/HDiffPatch/pulls)
 [![+issue Welcome](https://img.shields.io/github/issues-raw/sisong/HDiffPatch?color=green&label=%2Bissue%20welcome)](https://github.com/sisong/HDiffPatch/issues)   
@@ -109,7 +109,7 @@ special options:
       create diffFile compatible with bsdiff4, unsupport input directory(folder).
   -VCD[-compressLevel[-dictSize]]
       create diffFile compatible with VCDIFF, unsupport input directory(folder).
-      DEFAULT no compress, out format same as $open-vcdiff delta ... or $xdelta3 -S -e -n ...
+      DEFAULT no compress, out format same as $open-vcdiff ... or $xdelta3 -S -e -n ...
       if set compressLevel, out format same as $xdelta3 -S lzma -e -n ...
       compress by 7zXZ(xz), compressLevel in {0..9}, DEFAULT level 7;
       dictSize can like 4096 or 4k or 4m or 16m etc..., DEFAULT 8m

README_cmdline_cn.md

@@ -41,7 +41,7 @@
       创建一个和bsdiff4兼容的补丁, 不支持参数为文件夹。
   -VCD[-compressLevel[-dictSize]]
       创建一个标准规范VCDIFF格式的补丁, 不支持参数为文件夹。
-      默认输出补丁不带压缩, 格式和 $open-vcdiff delta ... 或 $xdelta3 -S -e -n ... 命令输出的补丁格式兼容；
+      默认输出补丁不带压缩, 格式和 $open-vcdiff ... 或 $xdelta3 -S -e -n ... 命令输出的补丁格式兼容；
       如果设置了压缩级别compressLevel, 那输出格式和 $xdelta3 -S lzma -e -n ...命令输出的补丁格式兼容；
       压缩输出时补丁文件使用7zXZ(xz)算法压缩, compressLevel可以选择0到9, 默认级别7；
       压缩字典大小dictSize可以设置为 4096, 4k, 4m, 16m等, 默认为8m

bsdiff_wrapper/bspatch_wrapper.c

@@ -46,6 +46,11 @@ static const char* kBsDiffVersionType = "BSDIFF40";
 #define            kBsDiffVersionTypeLen 8  // ==strlen(kBsDiffVersionType);
 #define            kBsDiffHeadLen       (kBsDiffVersionTypeLen+3*8)
 
+static const char* kEsBsDiffVersionType = "ENDSLEY/BSDIFF43";
+#define            kEsBsDiffVersionTypeLen 16  // ==strlen(kEsBsDiffVersionType);
+#define            kEsBsDiffHeadLen     (kEsBsDiffVersionTypeLen+8)
+
+
 static hpatch_inline hpatch_uint32_t _readUInt32(const unsigned char* buf){
     return buf[0] | (((hpatch_uint32_t)buf[1])<<8)  |
            (((hpatch_uint32_t)buf[2])<<16)  | (((hpatch_uint32_t)buf[3])<<24) ;
@@ -65,18 +70,31 @@ hpatch_BOOL getBsDiffInfo(hpatch_BsDiffInfo* out_diffinfo,const hpatch_TStreamIn
     unsigned char* buf=&_buf[0];
     if (diffStream->streamSize<kBsDiffHeadLen)
         return _hpatch_FALSE;
-    if (!diffStream->read(diffStream,0,buf,buf+kBsDiffHeadLen))
+    if (!diffStream->read(diffStream,0,buf,buf+kBsDiffHeadLen)) // must bz2 compressed size>=8
         return _hpatch_FALSE;
-    if (0!=memcmp(buf,kBsDiffVersionType,kBsDiffVersionTypeLen))
+    if (0==memcmp(buf,kBsDiffVersionType,kBsDiffVersionTypeLen)){
+        out_diffinfo->isEsBsd=hpatch_FALSE;
+        out_diffinfo->headSize=kBsDiffHeadLen;
+    }else if (0==memcmp(buf,kEsBsDiffVersionType,kEsBsDiffVersionTypeLen)){
+        out_diffinfo->isEsBsd=hpatch_TRUE;
+        out_diffinfo->headSize=kEsBsDiffHeadLen;
+    }else{
         return _hpatch_FALSE;
-    buf+=kBsDiffVersionTypeLen;
-    out_diffinfo->headSize=kBsDiffHeadLen;
-    out_diffinfo->ctrlDataSize=readUInt64(buf);
-    out_diffinfo->subDataSize =readUInt64(buf+8);
-    out_diffinfo->newDataSize =readUInt64(buf+8*2);
+    }
+    if (out_diffinfo->isEsBsd){
+        buf+=kEsBsDiffVersionTypeLen;
+        out_diffinfo->ctrlDataSize=0;
+        out_diffinfo->subDataSize =0;
+        out_diffinfo->newDataSize =readUInt64(buf);
+    }else{
+        buf+=kBsDiffVersionTypeLen;
+        out_diffinfo->ctrlDataSize=readUInt64(buf);
+        out_diffinfo->subDataSize =readUInt64(buf+8);
+        out_diffinfo->newDataSize =readUInt64(buf+8*2);
+    }
     return (out_diffinfo->ctrlDataSize<diffStream->streamSize)&&
            (out_diffinfo->subDataSize<diffStream->streamSize)&&
-           (kBsDiffHeadLen+out_diffinfo->ctrlDataSize+out_diffinfo->subDataSize<diffStream->streamSize);
+           (out_diffinfo->headSize+out_diffinfo->ctrlDataSize+out_diffinfo->subDataSize<diffStream->streamSize);
 }
 
 hpatch_BOOL getBsDiffInfo_mem(hpatch_BsDiffInfo* out_diffinfo,const unsigned char* diffData,const unsigned char* diffData_end){
@@ -175,6 +193,8 @@ hpatch_BOOL bspatch_with_cache(const hpatch_TStreamOutput* out_newData,
     hpatch_BsDiffInfo diffInfo;
     TStreamCacheClip  ctrlClip;
     TStreamCacheClip  subClip;
+    TStreamCacheClip* _ctrlClip=&ctrlClip;
+    TStreamCacheClip* _subClip=&subClip;
     TStreamCacheClip  newDataDiffClip;
     _TDecompressInputStream decompressers[3];
     hpatch_size_t           i;
@@ -206,29 +226,38 @@ hpatch_BOOL bspatch_with_cache(const hpatch_TStreamOutput* out_newData,
         temp_cache+=cacheSize;
         cacheSize=(temp_cache_end-temp_cache);
     }
-    cacheSize=cacheSize/_kCacheBsDecCount;
+    cacheSize=cacheSize/(diffInfo.isEsBsd?_kCacheBsDecCount-2:_kCacheBsDecCount);
     if (cacheSize<8) return _hpatch_FALSE;
 
     diffPos0=diffInfo.headSize;
-    if (!getStreamClip(&ctrlClip,&decompressers[0],
-                       _kUnknowMaxSize,diffInfo.ctrlDataSize,compressedDiff,&diffPos0,
-                       decompressPlugin,temp_cache,cacheSize)) _clear_return(_hpatch_FALSE);
-    temp_cache+=cacheSize;
-    if (!getStreamClip(&subClip,&decompressers[1],
-                       _kUnknowMaxSize,diffInfo.subDataSize,compressedDiff,&diffPos0,
-                       decompressPlugin,temp_cache,cacheSize)) _clear_return(_hpatch_FALSE);
-    temp_cache+=cacheSize;
-    if (!getStreamClip(&newDataDiffClip,&decompressers[2],
-                       _kUnknowMaxSize,compressedDiff->streamSize-diffPos0,compressedDiff,&diffPos0,
-                       decompressPlugin,temp_cache,cacheSize)) _clear_return(_hpatch_FALSE);
-    temp_cache+=cacheSize;
+    if (diffInfo.isEsBsd){
+        if (!getStreamClip(&newDataDiffClip,&decompressers[0],
+                        _kUnknowMaxSize,compressedDiff->streamSize-diffPos0,compressedDiff,&diffPos0,
+                        decompressPlugin,temp_cache,cacheSize)) _clear_return(_hpatch_FALSE);
+        temp_cache+=cacheSize;
+        _ctrlClip=&newDataDiffClip;
+        _subClip=&newDataDiffClip;
+    }else{
+        if (!getStreamClip(&ctrlClip,&decompressers[0],
+                        _kUnknowMaxSize,diffInfo.ctrlDataSize,compressedDiff,&diffPos0,
+                        decompressPlugin,temp_cache,cacheSize)) _clear_return(_hpatch_FALSE);
+        temp_cache+=cacheSize;
+        if (!getStreamClip(&subClip,&decompressers[1],
+                        _kUnknowMaxSize,diffInfo.subDataSize,compressedDiff,&diffPos0,
+                        decompressPlugin,temp_cache,cacheSize)) _clear_return(_hpatch_FALSE);
+        temp_cache+=cacheSize;
+        if (!getStreamClip(&newDataDiffClip,&decompressers[2],
+                        _kUnknowMaxSize,compressedDiff->streamSize-diffPos0,compressedDiff,&diffPos0,
+                        decompressPlugin,temp_cache,cacheSize)) _clear_return(_hpatch_FALSE);
+        temp_cache+=cacheSize;
+    }
     assert(diffPos0==compressedDiff->streamSize);
 
     {
         _TOutStreamCache outCache;
         _TOutStreamCache_init(&outCache,out_newData,temp_cache,cacheSize);
         temp_cache+=cacheSize;
-        result=bspatchByClip(&outCache,oldData,&ctrlClip,&subClip,&newDataDiffClip,
+        result=bspatchByClip(&outCache,oldData,_ctrlClip,_subClip,&newDataDiffClip,
                             temp_cache,cacheSize);
     }
 

bsdiff_wrapper/bspatch_wrapper.h

@@ -37,6 +37,7 @@ typedef struct hpatch_BsDiffInfo{
     hpatch_StreamPos_t  ctrlDataSize;
     hpatch_StreamPos_t  subDataSize;
     hpatch_StreamPos_t  newDataSize;
+    hpatch_BOOL         isEsBsd;
 } hpatch_BsDiffInfo;
 
 hpatch_BOOL getIsBsDiff(const hpatch_TStreamInput* diffData);

builds/android_ndk_jni_mk/Android.mk

@@ -5,11 +5,12 @@ LOCAL_MODULE := hpatchz
 
 # args
 ZSTD  := 1
+LZMA  := 1
+BROTLI:= 0
+VCD   := 0
 # if open BSD,must open BZIP2
 BSD   := 0
-VCD   := 0
 BZIP2 := 0
-LZMA  := 1
 
 ifeq ($(BZIP2),0)
   Bz2_Files :=
@@ -32,9 +33,9 @@ else
   LZMA_PATH  := $(LOCAL_PATH)/../../../lzma/C/
   Lzma_Files := $(LZMA_PATH)/LzmaDec.c \
                 $(LZMA_PATH)/Lzma2Dec.c
- ifeq ($(TARGET_ARCH_ABI),arm64-v8a)
-  Lzma_Files += $(LZMA_PATH)/../Asm/arm64/LzmaDecOpt.S
- endif
+  ifeq ($(TARGET_ARCH_ABI),arm64-v8a)
+    Lzma_Files += $(LZMA_PATH)/../Asm/arm64/LzmaDecOpt.S
+  endif
   ifeq ($(VCD),0)
   else
     Lzma_Files+=$(LZMA_PATH)/7zCrc.c \
@@ -70,6 +71,22 @@ else
   				$(ZSTD_PATH)/decompress/zstd_decompress_block.c
 endif
 
+ifeq ($(BROTLI),0)
+  Brotli_Files:=
+else
+  BROTLI_PATH := $(LOCAL_PATH)/../../../brotli/c/
+  Brotli_Files:=$(BROTLI_PATH)/common/constants.c \
+                $(BROTLI_PATH)/common/context.c \
+                $(BROTLI_PATH)/common/dictionary.c \
+                $(BROTLI_PATH)/common/platform.c \
+                $(BROTLI_PATH)/common/shared_dictionary.c \
+                $(BROTLI_PATH)/common/transform.c \
+                $(BROTLI_PATH)/dec/bit_reader.c \
+                $(BROTLI_PATH)/dec/decode.c \
+                $(BROTLI_PATH)/dec/huffman.c \
+                $(BROTLI_PATH)/dec/state.c
+endif
+
 HDP_PATH  := $(LOCAL_PATH)/../../
 Hdp_Files := $(HDP_PATH)/file_for_patch.c \
              $(HDP_PATH)/libHDiffPatch/HPatch/patch.c
@@ -79,8 +96,8 @@ else
 endif
 ifeq ($(VCD),0)
 else
-  Hdp_Files += $(HDP_PATH)/vcdiff_wrapper/vcpatch_wrapper.c
-  Hdp_Files += $(HDP_PATH)/libHDiffPatch/HDiff/private_diff/limit_mem_diff/adler_roll.c
+  Hdp_Files += $(HDP_PATH)/vcdiff_wrapper/vcpatch_wrapper.c \
+               $(HDP_PATH)/libHDiffPatch/HDiff/private_diff/limit_mem_diff/adler_roll.c
 endif
 
 Src_Files := $(LOCAL_PATH)/hpatch_jni.c \
@@ -120,6 +137,10 @@ else
 		-DDYNAMIC_BMI2=0 -DZSTD_LEGACY_SUPPORT=0 -DZSTD_LIB_DEPRECATED=0 -DHUF_FORCE_DECOMPRESS_X1=1 \
 		-DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT=1 -DZSTD_NO_INLINE=1 -DZSTD_STRIP_ERROR_STRINGS=1
 endif
+ifeq ($(BROTLI),0)
+else
+  DEF_FLAGS += -D_CompressPlugin_brotli -I$(BROTLI_PATH)/include
+endif
 
 LOCAL_SRC_FILES  := $(Src_Files) $(Bz2_Files) $(Lzma_Files) $(Zstd_Files) $(Hdp_Files)
 LOCAL_LDLIBS     := -llog -lz

builds/xcode/hpatchz.xcodeproj/project.pbxproj

@@ -274,6 +274,7 @@
 				CLANG_WARN_SUSPICIOUS_MOVE = YES;
 				CLANG_WARN_UNREACHABLE_CODE = YES;
 				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
+				CODE_SIGN_IDENTITY = "-";
 				ENABLE_STRICT_OBJC_MSGSEND = YES;
 				ENABLE_TESTABILITY = YES;
 				GCC_C_LANGUAGE_STANDARD = "compiler-default";
@@ -328,6 +329,7 @@
 				CLANG_WARN_SUSPICIOUS_MOVE = YES;
 				CLANG_WARN_UNREACHABLE_CODE = YES;
 				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
+				CODE_SIGN_IDENTITY = "-";
 				DEAD_CODE_STRIPPING = YES;
 				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
 				DEPLOYMENT_POSTPROCESSING = YES;
@@ -358,7 +360,6 @@
 			isa = XCBuildConfiguration;
 			buildSettings = {
 				CLANG_ENABLE_OBJC_WEAK = YES;
-				CODE_SIGN_IDENTITY = "-";
 				CODE_SIGN_STYLE = Automatic;
 				DEVELOPMENT_TEAM = "";
 				MACOSX_DEPLOYMENT_TARGET = 10.9;
@@ -371,7 +372,6 @@
 			isa = XCBuildConfiguration;
 			buildSettings = {
 				CLANG_ENABLE_OBJC_WEAK = YES;
-				CODE_SIGN_IDENTITY = "-";
 				CODE_SIGN_STYLE = Automatic;
 				DEVELOPMENT_TEAM = "";
 				MACOSX_DEPLOYMENT_TARGET = 10.9;

hdiffz.cpp

@@ -189,7 +189,7 @@ static void printUsage(){
 #   ifdef _CompressPlugin_7zXZ
            "  -VCD[-compressLevel[-dictSize]]\n"
            "      create diffFile compatible with VCDIFF, unsupport input directory(folder).\n"
-           "      DEFAULT no compress, out format same as $open-vcdiff delta ... or $xdelta3 -S -e -n ...\n"
+           "      DEFAULT no compress, out format same as $open-vcdiff ... or $xdelta3 -S -e -n ...\n"
            "      if set compressLevel, out format same as $xdelta3 -S lzma -e -n ...\n"
            "      compress by 7zXZ(xz), compressLevel in {0..9}, DEFAULT level 7;\n"
            "      dictSize can like 4096 or 4k or 4m or 16m etc..., DEFAULT 8m\n"
@@ -199,7 +199,7 @@ static void printUsage(){
 #   else
            "  -VCD\n"
            "      create diffFile compatible with VCDIFF, unsupport input directory(folder).\n"
-           "      out format same as $open-vcdiff delta ... or $xdelta3 -S -e -n ...\n"
+           "      out format same as $open-vcdiff ... or $xdelta3 -S -e -n ...\n"
 #   endif
            "      NOTE: out diffFile used large source window size!\n"
 #endif

hpatchz.c

@@ -987,7 +987,7 @@ int hpatch(const char* oldFileName,const char* diffFileName,const char* outNewFi
                 *decompressPlugin=_bz2DecompressPlugin_unsz;
                 decompressPlugin->decError=hpatch_dec_ok;
                 memcpy(diffInfo.compressType,bsCompressType,strlen(bsCompressType)+1);
-                diffInfo.compressedCount=3;
+                diffInfo.compressedCount=bsdiffInfo.isEsBsd?1:3;
                 diffInfo.newDataSize=bsdiffInfo.newDataSize;
                 diffInfo.oldDataSize=poldData->streamSize; //not saved oldDataSize
                 isBsDiff=hpatch_TRUE;

libHDiffPatch/HPatch/patch_types.h

@@ -38,7 +38,7 @@ extern "C" {
 
 #define HDIFFPATCH_VERSION_MAJOR    4
 #define HDIFFPATCH_VERSION_MINOR    6
-#define HDIFFPATCH_VERSION_RELEASE  6
+#define HDIFFPATCH_VERSION_RELEASE  7
 
 #define _HDIFFPATCH_VERSION          HDIFFPATCH_VERSION_MAJOR.HDIFFPATCH_VERSION_MINOR.HDIFFPATCH_VERSION_RELEASE
 #define _HDIFFPATCH_QUOTE(str) #str

vcdiff_wrapper/vcdiff_wrapper.cpp

@@ -33,10 +33,20 @@
 #include "../libHDiffPatch/HPatch/patch.h"
 #include <stdexcept>  //std::runtime_error
 #include <algorithm>  //std::inplace_merge
+#include <string>
 #define _check(value,info) { if (!(value)) { throw std::runtime_error(info); } }
 static const hpatch_byte kVcDiffType[3]={('V'|(1<<7)),('C'|(1<<7)),('D'|(1<<7))};
 #define kVcDiffVersion 0
-static const char* kHDiffzAppHead_a="$hdiffz-VCDIFF&7zXZ#a";
+
+static const char* kHDiffzAppHead="$hdiffz-VCDIFF&";
+static const char* kHDiffzAppHead_version="#a";
+
+static std::string getHDiffzAppHead(const vcdiff_TCompress* compressPlugin){
+    std::string str(kHDiffzAppHead);
+    if (compressPlugin&&compressPlugin->compress)
+        str+=compressPlugin->compress->compressType();
+    return str+kHDiffzAppHead_version;
+}
 
 namespace hdiff_private{
 
@@ -370,18 +380,18 @@ static void serialize_vcdiff(const hpatch_TStreamInput* newData,const hpatch_TSt
         pushBack(buf,kVcDiffType,sizeof(kVcDiffType));
         buf.push_back(kVcDiffVersion);
         const bool isHaveCompresser=(compressPlugin!=0)&&(compressPlugin->compress_type!=kVcDiff_compressorID_no);
-        const bool isHDiffzAppHead_a=isHaveCompresser;
-        hpatch_byte Hdr_Indicator = ((isHaveCompresser?1:0)<<0)     // VCD_DECOMPRESS
-                                  | (0<<1) // VCD_CODETABLE, no custom code table
-                                  | ((isHDiffzAppHead_a?1:0)<<2); // VCD_APPHEADER
+        if (!isHaveCompresser)
+            compressPlugin=0;
+        hpatch_byte Hdr_Indicator = ((isHaveCompresser?1:0)<<0) // VCD_DECOMPRESS
+                                  | (0<<1)  // VCD_CODETABLE, no custom code table
+                                  | (1<<2); // VCD_APPHEADER
         buf.push_back(Hdr_Indicator); 
-        if (isHaveCompresser)
+        if (isHaveCompresser)//VCD_DECOMPRESS
             buf.push_back(compressPlugin->compress_type);
-        else
-            compressPlugin=0;
-        if (isHDiffzAppHead_a){
-            packUInt(buf,strlen(kHDiffzAppHead_a));
-            pushCStr(buf,kHDiffzAppHead_a);
+        { //VCD_APPHEADER   // always add HDiffzAppHead tag to out_diff
+            const std::string HDiffzAppHead=getHDiffzAppHead(compressPlugin);
+            packUInt(buf,HDiffzAppHead.size());
+            pushCStr(buf,HDiffzAppHead.c_str());
         }
         _flushBuf(outDiff,buf);
     }