1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287
|
From: Rusty Russell <rusty@rustcorp.com.au>
[NOTE: this patch is for _gzip_! --Wayne]
It's pending for actual gzip inclusion! The maintainer was dug up and
everything: it'll even be on by default.
But here's the old patch,
Rusty.
--
Anyone who quotes me in their sig is an idiot. -- Rusty Russell.
--- gzip-1.2.4.orig/deflate.c Fri Aug 13 22:35:31 1993
+++ gzip-1.2.4-rsync/deflate.c Sat Dec 30 15:33:25 2000
@@ -121,6 +121,14 @@
#endif
/* Matches of length 3 are discarded if their distance exceeds TOO_FAR */
+#ifndef RSYNC_WIN
+# define RSYNC_WIN 4096
+#endif
+/* Size of rsync window, must be < MAX_DIST */
+
+#define RSYNC_SUM_MATCH(sum) ((sum) % RSYNC_WIN == 0)
+/* Whether window sum matches magic value */
+
/* ===========================================================================
* Local data used by the "longest match" routines.
*/
@@ -202,6 +210,8 @@
unsigned near good_match;
/* Use a faster search when the previous match is longer than this */
+local ulg rsync_sum; /* rolling sum of rsync window */
+local ulg rsync_chunk_end; /* next rsync sequence point */
/* Values for max_lazy_match, good_match and max_chain_length, depending on
* the desired pack level (0..9). The values given below have been tuned to
@@ -300,6 +310,10 @@
#endif
/* prev will be initialized on the fly */
+ /* rsync params */
+ rsync_chunk_end = 0xFFFFFFFFUL;
+ rsync_sum = 0;
+
/* Set the default configuration parameters:
*/
max_lazy_match = configuration_table[pack_level].max_lazy;
@@ -536,6 +550,8 @@
memcpy((char*)window, (char*)window+WSIZE, (unsigned)WSIZE);
match_start -= WSIZE;
strstart -= WSIZE; /* we now have strstart >= MAX_DIST: */
+ if (rsync_chunk_end != 0xFFFFFFFFUL)
+ rsync_chunk_end -= WSIZE;
block_start -= (long) WSIZE;
@@ -563,13 +579,46 @@
}
}
+local void rsync_roll(start, num)
+ unsigned start;
+ unsigned num;
+{
+ unsigned i;
+
+ if (start < RSYNC_WIN) {
+ /* before window fills. */
+ for (i = start; i < RSYNC_WIN; i++) {
+ if (i == start + num) return;
+ rsync_sum += (ulg)window[i];
+ }
+ num -= (RSYNC_WIN - start);
+ start = RSYNC_WIN;
+ }
+
+ /* buffer after window full */
+ for (i = start; i < start+num; i++) {
+ /* New character in */
+ rsync_sum += (ulg)window[i];
+ /* Old character out */
+ rsync_sum -= (ulg)window[i - RSYNC_WIN];
+ if (rsync_chunk_end == 0xFFFFFFFFUL && RSYNC_SUM_MATCH(rsync_sum))
+ rsync_chunk_end = i;
+ }
+}
+
+/* ===========================================================================
+ * Set rsync_chunk_end if window sum matches magic value.
+ */
+#define RSYNC_ROLL(s, n) \
+ do { if (rsync) rsync_roll((s), (n)); } while(0)
+
/* ===========================================================================
* Flush the current block, with given end-of-file flag.
* IN assertion: strstart is set to the end of the current match.
*/
#define FLUSH_BLOCK(eof) \
flush_block(block_start >= 0L ? (char*)&window[(unsigned)block_start] : \
- (char*)NULL, (long)strstart - block_start, (eof))
+ (char*)NULL, (long)strstart - block_start, flush-1, (eof))
/* ===========================================================================
* Processes a new input file and return its compressed length. This
@@ -580,7 +629,7 @@
local ulg deflate_fast()
{
IPos hash_head; /* head of the hash chain */
- int flush; /* set if current block must be flushed */
+ int flush; /* set if current block must be flushed, 2=>and padded */
unsigned match_length = 0; /* length of best match */
prev_length = MIN_MATCH-1;
@@ -609,6 +658,7 @@
lookahead -= match_length;
+ RSYNC_ROLL(strstart, match_length);
/* Insert new strings in the hash table only if the match length
* is not too large. This saves time but degrades compression.
*/
@@ -637,9 +687,14 @@
/* No match, output a literal byte */
Tracevv((stderr,"%c",window[strstart]));
flush = ct_tally (0, window[strstart]);
+ RSYNC_ROLL(strstart, 1);
lookahead--;
strstart++;
}
+ if (rsync && strstart > rsync_chunk_end) {
+ rsync_chunk_end = 0xFFFFFFFFUL;
+ flush = 2;
+ }
if (flush) FLUSH_BLOCK(0), block_start = strstart;
/* Make sure that we always have enough lookahead, except
@@ -715,6 +770,7 @@
*/
lookahead -= prev_length-1;
prev_length -= 2;
+ RSYNC_ROLL(strstart, prev_length+1);
do {
strstart++;
INSERT_STRING(strstart, hash_head);
@@ -727,24 +783,39 @@
match_available = 0;
match_length = MIN_MATCH-1;
strstart++;
- if (flush) FLUSH_BLOCK(0), block_start = strstart;
+ if (rsync && strstart > rsync_chunk_end) {
+ rsync_chunk_end = 0xFFFFFFFFUL;
+ flush = 2;
+ }
+ if (flush) FLUSH_BLOCK(0), block_start = strstart;
} else if (match_available) {
/* If there was no match at the previous position, output a
* single literal. If there was a match but the current match
* is longer, truncate the previous match to a single literal.
*/
Tracevv((stderr,"%c",window[strstart-1]));
- if (ct_tally (0, window[strstart-1])) {
- FLUSH_BLOCK(0), block_start = strstart;
- }
+ flush = ct_tally (0, window[strstart-1]);
+ if (rsync && strstart > rsync_chunk_end) {
+ rsync_chunk_end = 0xFFFFFFFFUL;
+ flush = 2;
+ }
+ if (flush) FLUSH_BLOCK(0), block_start = strstart;
+ RSYNC_ROLL(strstart, 1);
strstart++;
lookahead--;
} else {
/* There is no previous match to compare with, wait for
* the next step to decide.
*/
+ if (rsync && strstart > rsync_chunk_end) {
+ /* Reset huffman tree */
+ rsync_chunk_end = 0xFFFFFFFFUL;
+ flush = 2;
+ FLUSH_BLOCK(0), block_start = strstart;
+ }
match_available = 1;
+ RSYNC_ROLL(strstart, 1);
strstart++;
lookahead--;
}
--- gzip-1.2.4.orig/gzip.c Thu Aug 19 23:39:43 1993
+++ gzip-1.2.4-rsync/gzip.c Fri Dec 29 21:20:54 2000
@@ -239,6 +239,7 @@
unsigned insize; /* valid bytes in inbuf */
unsigned inptr; /* index of next byte to be processed in inbuf */
unsigned outcnt; /* bytes in output buffer */
+int rsync = 0; /* make ryncable chunks */
struct option longopts[] =
{
@@ -268,6 +269,7 @@
{"best", 0, 0, '9'}, /* compress better */
{"lzw", 0, 0, 'Z'}, /* make output compatible with old compress */
{"bits", 1, 0, 'b'}, /* max number of bits per code (implies -Z) */
+ {"rsyncable", 0, 0, 'R'}, /* make rsync-friendly archive */
{ 0, 0, 0, 0 }
};
@@ -357,6 +359,7 @@
" -Z --lzw produce output compatible with old compress",
" -b --bits maxbits max number of bits per code (implies -Z)",
#endif
+ " --rsyncable Make rsync-friendly archive",
" file... files to (de)compress. If none given, use standard input.",
0};
char **p = help_msg;
@@ -516,6 +519,9 @@
#else
recursive = 1; break;
#endif
+ case 'R':
+ rsync = 1; break;
+
case 'S':
#ifdef NO_MULTIPLE_DOTS
if (*optarg == '.') optarg++;
--- gzip-1.2.4.orig/gzip.h Fri Aug 13 22:35:33 1993
+++ gzip-1.2.4-rsync/gzip.h Sat Dec 30 15:26:56 2000
@@ -131,6 +131,7 @@
extern unsigned insize; /* valid bytes in inbuf */
extern unsigned inptr; /* index of next byte to be processed in inbuf */
extern unsigned outcnt; /* bytes in output buffer */
+extern int rsync; /* deflate into rsyncable chunks */
extern long bytes_in; /* number of input bytes */
extern long bytes_out; /* number of output bytes */
@@ -282,7 +283,7 @@
/* in trees.c */
void ct_init OF((ush *attr, int *method));
int ct_tally OF((int dist, int lc));
-ulg flush_block OF((char *buf, ulg stored_len, int eof));
+ulg flush_block OF((char *buf, ulg stored_len, int pad, int eof));
/* in bits.c */
void bi_init OF((file_t zipfile));
--- gzip-1.2.4.orig/gzip.texi Thu Aug 19 06:42:50 1993
+++ gzip-1.2.4-rsync/gzip.texi Fri Dec 29 21:20:54 2000
@@ -316,6 +316,14 @@
into the directory and compress all the files it finds there (or
decompress them in the case of @code{gunzip}).
+@item --rsyncable
+While compressing, synchronize the output occasionally based on the
+input. This reduces compression by about 1 percent most cases, but
+means that the @code{rsync} program can take advantage of similarities
+in the uncompressed input when syncronizing two files compressed with
+this flag. @code{gunzip} cannot tell the difference between a
+compressed file created with this option, and one created without it.
+
@item --suffix @var{suf}
@itemx -S @var{suf}
Use suffix @samp{@var{suf}} instead of @samp{.gz}. Any suffix can be
--- gzip-1.2.4.orig/trees.c Wed Aug 18 03:36:32 1993
+++ gzip-1.2.4-rsync/trees.c Sat Dec 30 15:37:00 2000
@@ -850,9 +850,10 @@
* trees or store, and output the encoded block to the zip file. This function
* returns the total compressed length for the file so far.
*/
-ulg flush_block(buf, stored_len, eof)
+ulg flush_block(buf, stored_len, pad, eof)
char *buf; /* input block, or NULL if too old */
ulg stored_len; /* length of input block */
+ int pad; /* pad output to byte boundary */
int eof; /* true if this is the last block for a file */
{
ulg opt_lenb, static_lenb; /* opt_len and static_len in bytes */
@@ -944,6 +967,10 @@
Assert (input_len == isize, "bad input size");
bi_windup();
compressed_len += 7; /* align on byte boundary */
+ } else if (pad && (compressed_len % 8) != 0) {
+ send_bits((STORED_BLOCK<<1)+eof, 3); /* send block type */
+ compressed_len = (compressed_len + 3 + 7) & ~7L;
+ copy_block(buf, 0, 1); /* with header */
}
Tracev((stderr,"\ncomprlen %lu(%lu) ", compressed_len>>3,
compressed_len-7*eof));
|