Skip to content

Commit 680c86d

Browse files
sisongsisong
authored
Merge pull request #319 from sisong/dev
recode some default settings for cmdline
2 parents 492b1b7 + 7c42c9d commit 680c86d

File tree

17 files changed

+271
-203
lines changed

17 files changed

+271
-203
lines changed

‎README.md‎

Lines changed: 75 additions & 77 deletions
Large diffs are not rendered by default.

‎README_cmdline_cn.md‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,11 @@
2222
匹配块大小matchBlockSize>=4, 默认为64, 推荐16,32,48,1k,64k,1m等;
2323
一般匹配块越大,内存占用越小,速度越快,但补丁包可能变大。
2424
其他选项:
25-
-block[-fastMatchBlockSize]
25+
-block-fastMatchBlockSize
2626
必须和-m配合使用;
27-
在使用较慢的匹配之前使用基于块的快速匹配, 默认不开启;
28-
快速块匹配大小fastMatchBlockSize>=4, 默认为4k, 推荐256,1k,64k,1m等;
27+
在使用较慢的逐字节匹配之前使用基于块的快速匹配, 默认-block-4k;
28+
如果设置为-block-0，意思是关闭基于块的提前匹配；
29+
快速块匹配大小fastMatchBlockSize>=4, 推荐256,1k,64k,1m等;
2930
如果新版本和旧版本相同数据比较多,那diff速度就会比较快,并且减少内存占用,
3031
但有很小的可能补丁包会变大。
3132
-cache
@@ -45,15 +46,12 @@
4546
设置补丁数据使用的压缩算法和压缩级别等, 默认不压缩;
4647
补丁另存时,使用新的压缩参数设置来输出新补丁;
4748
支持的压缩算法、压缩级别和字典大小等:
48-
(参考 https://github.com/sisong/lzbench/blob/master/lzbench171_sorted.md )
4949
-c-zlib[-{1..9}[-dictBits]] 默认级别 9
5050
压缩字典比特数dictBits可以为9到15, 默认为15。
51-
-c-pzlib[-{1..9}[-dictBits]] 默认级别 6
52-
压缩字典比特数dictBits可以为9到15, 默认为15。
5351
支持多线程并行压缩,很快！
5452
-c-bzip2[-{1..9}] (或 -bz2) 默认级别 9
5553
-c-pbzip2[-{1..9}] (或 -pbz2) 默认级别 8
56-
支持并行压缩,生成的补丁和-c-bzip2的输出格式不同,一般也会更大一点。
54+
支持并行压缩,生成的补丁和-c-bzip2的输出格式不同,一般也可能稍大一点。
5755
-c-lzma[-{0..9}[-dictSize]] 默认级别 7
5856
压缩字典大小dictSize可以设置为 4096, 4k, 4m, 128m等, 默认为8m
5957
支持2个线程并行压缩。
@@ -119,7 +117,7 @@
119117
```
120118
内存选项:
121119
-s[-cacheSize]
122-
默认选项,并且设置为-s-64m; oldPath所有文件被当作文件流来加载;
120+
默认选项,并且默认设置为-s-4m; oldPath所有文件被当作文件流来加载;
123121
cacheSize可以设置为262144 或 256k, 512m, 2g等
124122
需要的内存大小: (cacheSize + 4*解压缩缓冲区)+O(1)
125123
而如果diffFile是单压缩流的补丁文件

‎builds/android_ndk_jni_mk/hpatch_jni.c‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,17 +12,25 @@ extern "C" {
1212
#define _check_jn2cstr(jstr,cstr) do { if (jstr) __j2cstr_(jstr,cstr); else (cstr)=0; } while(0)
1313
#define _jrelease_cstr(jstr,cstr) do { if (cstr) (*jenv)->ReleaseStringUTFChars(jenv,jstr,cstr); } while(0)
1414

15+
16+
static size_t getCacheMemory(jlong cacheMemory){
17+
#define kPatchCacheSize_default (1024*256)
18+
#define kPatchCacheSize_max ((jlong)((size_t)(~(size_t)0)))
19+
if (cacheMemory<0) return kPatchCacheSize_default;
20+
if (sizeof(jlong)<=sizeof(size_t)) return (size_t)cacheMemory;
21+
return (size_t)((cacheMemory<kPatchCacheSize_max)?cacheMemory:kPatchCacheSize_max);
22+
}
23+
1524
JNIEXPORT int
1625
Java_com_github_sisong_HPatch_patch(JNIEnv* jenv,jobject jobj,
1726
jstring oldFileName,jstring diffFileName,
1827
jstring outNewFileName,jlong cacheMemory){
1928
const char* cOldFileName =0;
2029
const char* cDiffFileName =0;
2130
const char* cOutNewFileName=0;
22-
size_t cCacheMemory=(size_t)cacheMemory;
31+
size_t cCacheMemory=getCacheMemory(cacheMemory);
2332
int result=0;
2433

25-
_check_rt((jlong)cCacheMemory==cacheMemory);
2634
_check_jn2cstr(oldFileName,cOldFileName);
2735
_check_j2cstr(diffFileName,cDiffFileName);
2836
_check_j2cstr(outNewFileName,cOutNewFileName);

‎builds/android_ndk_jni_mk/java/com/github/sisong/HPatch.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,14 @@ public class HPatch{
44
// return THPatchResult, 0 is ok
55
// 'diffFileName' file is create by hdiffz app,or by create_compressed_diff(),create_compressed_diff_stream()
66
// or by create_single_compressed_diff(),create_single_compressed_diff_stream()
7-
// cacheMemory recommended 256*1024 1024*1024...
7+
// cacheMemory recommended 256*1024,1024*1024,... if cacheMemory<0 then default 256*1024
88
public static native int patch(String oldFileName,String diffFileName,
99
String outNewFileName,long cacheMemory);
10+
11+
public static int patch(String oldFileName,String diffFileName,String outNewFileName){
12+
return patch(oldFileName,diffFileName,outNewFileName,-1);
13+
}
14+
15+
// ? auto load libhpatchz.so
16+
//static { System.loadLibrary("hpatchz"); }
1017
}

‎compress_parallel.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
#include "libParallel/parallel_channel.h"
3434
#include "libHDiffPatch/HDiff/private_diff/mem_buf.h"
3535

36+
namespace{
3637
struct TMt_base {
3738
CChannel work_chan;
3839

@@ -104,6 +105,7 @@ struct _auto_thread_end_t{
104105
inline ~_auto_thread_end_t() { _mt.thread_end(); }
105106
TMt_base& _mt;
106107
};
108+
}
107109

108110
#ifdef __cplusplus
109111
extern "C" {

‎compress_plugin_demo.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -365,7 +365,7 @@ int _default_setParallelThreadNumber(hdiff_TCompress* compressPlugin,int threadN
365365
}
366366
}
367367

368-
_def_fun_compressType(_pzlib_compressType,"pzlib");
368+
_def_fun_compressType(_pzlib_compressType,"zlib"); // pzlibCompressPlugin now out standard deflate code, same as zlibCompressPlugin
369369
static const TCompressPlugin_pzlib pzlibCompressPlugin={
370370
{ {_pzlib_compressType,_default_maxCompressedSize,_pzlib_setThreadNum,_pzlib_compress},
371371
6,8,-MAX_WBITS,hpatch_TRUE,Z_DEFAULT_STRATEGY},

‎dirDiffPatch/dir_diff/dir_diff.h‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,18 @@
2929
#ifndef hdiff_dir_diff_h
3030
#define hdiff_dir_diff_h
3131
#include "../dir_patch/dir_patch_types.h"
32+
33+
struct THDiffSets{
34+
hpatch_BOOL isDiffInMem;//or diff by stream
35+
hpatch_BOOL isSingleCompressedDiff;
36+
//diff in mem
37+
hpatch_BOOL isUseBigCacheMatch;
38+
size_t matchScore;
39+
size_t patchStepMemSize;
40+
size_t matchBlockSize;
41+
size_t threadNum;
42+
};
43+
3244
#if (_IS_NEED_DIR_DIFF_PATCH)
3345
#include "dir_manifest.h"
3446

‎dirDiffPatch/dir_patch/dir_patch_types.h‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -32,17 +32,6 @@
3232
#ifdef __cplusplus
3333
extern "C" {
3434
#endif
35-
36-
struct THDiffSets{
37-
hpatch_BOOL isDiffInMem;//or diff by stream
38-
hpatch_BOOL isSingleCompressedDiff;
39-
//diff in mem
40-
hpatch_BOOL isUseBigCacheMatch;
41-
size_t matchScore;
42-
size_t patchStepMemSize;
43-
size_t matchBlockSize;
44-
size_t threadNum;
45-
};
4635

4736
#ifndef _IS_NEED_DIR_DIFF_PATCH
4837
# define _IS_NEED_DIR_DIFF_PATCH 1

‎hdiffz.cpp‎

Lines changed: 20 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -153,10 +153,11 @@ static void printUsage(){
153153
")bytes of memory;\n"
154154
" matchBlockSize>=4, DEFAULT -s-64, recommended 16,32,48,1k,64k,1m etc...\n"
155155
"special options:\n"
156-
" -block[-fastMatchBlockSize] \n"
156+
" -block-fastMatchBlockSize \n"
157157
" must run with -m;\n"
158-
" set is use fast block match befor slow match, DEFAULT false;\n"
159-
" fastMatchBlockSize>=4, DEFAULT 4k, recommended 256,1k,64k,1m etc...;\n"
158+
" set block match befor slow byte-by-byte match, DEFAULT -block-4k;\n"
159+
" if set -block-0, means don't use block match;\n"
160+
" fastMatchBlockSize>=4, recommended 256,1k,64k,1m etc...\n"
160161
" if newData similar to oldData then diff speed++ & diff memory--,\n"
161162
" but small possibility outDiffFile's size+\n"
162163
" -cache \n"
@@ -181,13 +182,10 @@ static void printUsage(){
181182
" set outDiffFile Compress type, DEFAULT uncompress;\n"
182183
" for resave diffFile,recompress diffFile to outDiffFile by new set;\n"
183184
" support compress type & level & dict:\n"
184-
" (re. https://github.com/sisong/lzbench/blob/master/lzbench171_sorted.md )\n"
185185
#ifdef _CompressPlugin_zlib
186186
" -c-zlib[-{1..9}[-dictBits]] DEFAULT level 9\n"
187187
" dictBits can 9--15, DEFAULT 15.\n"
188188
# if (_IS_USED_MULTITHREAD)
189-
" -c-pzlib[-{1..9}[-dictBits]] DEFAULT level 6\n"
190-
" dictBits can 9--15, DEFAULT 15.\n"
191189
" support run by multi-thread parallel, fast!\n"
192190
# endif
193191
#endif
@@ -591,16 +589,14 @@ static int _checkSetCompress(hdiff_TCompress** out_compressPlugin,
591589
const size_t defaultDictBits_zlib=15; //32k
592590
#endif
593591
#ifdef _CompressPlugin_zlib
594-
__getCompressSet(_tryGetCompressSet(&isMatchedType,ptype,ptypeEnd,"zlib",0,
592+
__getCompressSet(_tryGetCompressSet(&isMatchedType,ptype,ptypeEnd,"zlib","pzlib",
595593
&compressLevel,1,9,9, &dictBits,9,15,defaultDictBits_zlib),"-c-zlib-?"){
594+
# if (!_IS_USED_MULTITHREAD)
596595
static TCompressPlugin_zlib _zlibCompressPlugin=zlibCompressPlugin;
597596
_zlibCompressPlugin.compress_level=(int)compressLevel;
598597
_zlibCompressPlugin.windowBits=(signed char)(-dictBits);
599598
*out_compressPlugin=&_zlibCompressPlugin.base; }}
600-
# if (_IS_USED_MULTITHREAD)
601-
//pzlib
602-
__getCompressSet(_tryGetCompressSet(&isMatchedType,ptype,ptypeEnd,"pzlib",0,
603-
&compressLevel,1,9,6, &dictBits,9,15,defaultDictBits_zlib),"-c-pzlib-?"){
599+
# else
604600
static TCompressPlugin_pzlib _pzlibCompressPlugin=pzlibCompressPlugin;
605601
_pzlibCompressPlugin.base.compress_level=(int)compressLevel;
606602
_pzlibCompressPlugin.base.windowBits=(signed char)(-dictBits);
@@ -716,7 +712,7 @@ int hdiff_cmd_line(int argc, const char * argv[]){
716712
diffSets.isDiffInMem =_kNULL_VALUE;
717713
diffSets.isSingleCompressedDiff =_kNULL_VALUE;
718714
diffSets.isUseBigCacheMatch =_kNULL_VALUE;
719-
diffSets.matchBlockSize=0;
715+
diffSets.matchBlockSize=_kNULL_SIZE;
720716
diffSets.threadNum=_THREAD_NUMBER_NULL;
721717
hpatch_BOOL isForceOverwrite=_kNULL_VALUE;
722718
hpatch_BOOL isOutputHelp=_kNULL_VALUE;
@@ -771,12 +767,13 @@ int hdiff_cmd_line(int argc, const char * argv[]){
771767
} break;
772768
case 's':{
773769
_options_check((diffSets.isDiffInMem==_kNULL_VALUE),"-s");
774-
_options_check((diffSets.matchBlockSize==0),"-block must run with -m");
770+
_options_check((diffSets.matchBlockSize==_kNULL_SIZE),"-block must run with -m");
775771
diffSets.isDiffInMem=hpatch_FALSE; //diff by stream
776772
if (op[2]=='-'){
777773
const char* pnum=op+3;
778774
_options_check(kmg_to_size(pnum,strlen(pnum),&diffSets.matchBlockSize),"-s-?");
779-
_options_check(kMatchBlockSize_min<=diffSets.matchBlockSize,"-s-?");
775+
_options_check((kMatchBlockSize_min<=diffSets.matchBlockSize)
776+
&&(diffSets.matchBlockSize!=_kNULL_SIZE),"-s-?");
780777
}else{
781778
diffSets.matchBlockSize=kMatchBlockSize_default;
782779
}
@@ -832,13 +829,15 @@ int hdiff_cmd_line(int argc, const char * argv[]){
832829
} break;
833830
#endif
834831
case 'b':{
835-
_options_check((diffSets.matchBlockSize==0)&&
832+
_options_check((diffSets.matchBlockSize==_kNULL_SIZE)&&
836833
(op[2]=='l')&&(op[3]=='o')&&(op[4]=='c')&&(op[5]=='k')&&
837834
((op[6]=='\0')||(op[6]=='-')),"-block?");
838835
if (op[6]=='-'){
839836
const char* pnum=op+7;
840837
_options_check(kmg_to_size(pnum,strlen(pnum),&diffSets.matchBlockSize),"-block-?");
841-
_options_check(kMatchBlockSize_min<=diffSets.matchBlockSize,"-block-?");
838+
if (diffSets.matchBlockSize!=0)
839+
_options_check((kMatchBlockSize_min<=diffSets.matchBlockSize)
840+
&&(diffSets.matchBlockSize!=_kNULL_SIZE),"-block-?");
842841
}else{
843842
diffSets.matchBlockSize=kDefaultFastMatchBlockSize;
844843
}
@@ -867,9 +866,10 @@ int hdiff_cmd_line(int argc, const char * argv[]){
867866
_options_check(_getOptChecksum(&checksumPlugin,ptype,"no"),"-C-?");
868867
} break;
869868
case 'n':{
870-
_options_check((kMaxOpenFileNumber==_kNULL_SIZE)&&(op[2]=='-'),"-n-?")
869+
_options_check((kMaxOpenFileNumber==_kNULL_SIZE)&&(op[2]=='-'),"-n")
871870
const char* pnum=op+3;
872871
_options_check(kmg_to_size(pnum,strlen(pnum),&kMaxOpenFileNumber),"-n-?");
872+
_options_check((kMaxOpenFileNumber!=_kNULL_SIZE),"-n-?");
873873
} break;
874874
case 'g':{
875875
if (op[2]=='#'){ //-g#
@@ -954,6 +954,8 @@ int hdiff_cmd_line(int argc, const char * argv[]){
954954
if (kMaxOpenFileNumber<kMaxOpenFileNumber_default_min)
955955
kMaxOpenFileNumber=kMaxOpenFileNumber_default_min;
956956
#endif
957+
if (diffSets.isDiffInMem&&(diffSets.matchBlockSize==_kNULL_SIZE))
958+
diffSets.matchBlockSize=kDefaultFastMatchBlockSize;
957959
if (diffSets.threadNum==_THREAD_NUMBER_NULL)
958960
diffSets.threadNum=_THREAD_NUMBER_DEFUALT;
959961
else if (diffSets.threadNum>_THREAD_NUMBER_MAX)
@@ -1024,7 +1026,7 @@ int hdiff_cmd_line(int argc, const char * argv[]){
10241026
#endif
10251027
{
10261028
diffSets.isDiffInMem=hpatch_FALSE; //not need -m, set as -s
1027-
diffSets.matchBlockSize=hpatch_kStreamCacheSize; //not used
1029+
diffSets.matchBlockSize=kDefaultFastMatchBlockSize; //not used
10281030
diffSets.isUseBigCacheMatch=hpatch_FALSE;
10291031
}
10301032
}else{

0 commit comments

Comments
 (0)