Merging upstream version 0.7.

Signed-off-by: Daniel Baumann <daniel@debian.org>

ChangeLog

@@ -1,3 +1,10 @@
+2009-10-20  Antonio Diaz Diaz  <ant_diaz@teleline.es>
+
+	* Version 0.7 released.
+	* Compression time has been reduced by 4%.
+	* testsuite/check.sh: Removed -9 to run in less than 256MiB of RAM.
+	* lzcheck.cc: Read files of any size up to 2^63 bytes.
+
 2009-09-02  Antonio Diaz Diaz  <ant_diaz@teleline.es>
 
 	* Version 0.6 released.

Makefile.in

@@ -74,7 +74,8 @@ $(VPATH)/doc/$(pkgname).info : $(VPATH)/doc/$(pkgname).texinfo
 man : $(VPATH)/doc/$(progname).1
 
 $(VPATH)/doc/$(progname).1 : $(progname)
-	help2man -o $(VPATH)/doc/$(progname).1 --no-info ./$(progname)
+	help2man -n 'test program for the lzlib library' \
+	  -o $(VPATH)/doc/$(progname).1 --no-info ./$(progname)
 
 Makefile : $(VPATH)/configure $(VPATH)/Makefile.in
 	./config.status
@@ -83,27 +84,27 @@ check : all $(VPATH)/testsuite/check.sh lzcheck
 	@$(VPATH)/testsuite/check.sh $(VPATH)/testsuite
 
 install : all install-info
-	if test ! -d $(DESTDIR)$(includedir) ; then $(INSTALL_DIR) $(DESTDIR)$(includedir) ; fi
+	if [ ! -d $(DESTDIR)$(includedir) ] ; then $(INSTALL_DIR) $(DESTDIR)$(includedir) ; fi
-	if test ! -d $(DESTDIR)$(libdir) ; then $(INSTALL_DIR) $(DESTDIR)$(libdir) ; fi
+	if [ ! -d $(DESTDIR)$(libdir) ] ; then $(INSTALL_DIR) $(DESTDIR)$(libdir) ; fi
 	$(INSTALL_DATA) $(VPATH)/$(pkgname).h $(DESTDIR)$(includedir)/$(pkgname).h
 	$(INSTALL_DATA) ./$(libname).a $(DESTDIR)$(libdir)/$(libname).a
-	if test -n "$(progname_shared)" ; then \
+	if [ -n "$(progname_shared)" ] ; then \
 	  $(INSTALL_PROGRAM) ./$(libname).so.$(pkgversion) $(DESTDIR)$(libdir)/$(libname).so.$(pkgversion) ; \
-	  if test -a $(DESTDIR)$(libdir)/$(libname).so.$(soversion) ; then \
+	  if [ -e $(DESTDIR)$(libdir)/$(libname).so.$(soversion) ] ; then \
 	    run_ldconfig=no ; rm -f $(DESTDIR)$(libdir)/$(libname).so.$(soversion) ; \
 	  else run_ldconfig=yes ; \
 	  fi ; \
 	  cd $(DESTDIR)$(libdir) && ln -s $(libname).so.$(pkgversion) $(libname).so.$(soversion) ; \
-	  if test $${run_ldconfig} = yes ; then $(LDCONFIG) $(DESTDIR)$(libdir) ; fi ; \
+	  if [ $${run_ldconfig} = yes ] ; then $(LDCONFIG) $(DESTDIR)$(libdir) ; fi ; \
 	fi
 
 install-info :
-	if test ! -d $(DESTDIR)$(infodir) ; then $(INSTALL_DIR) $(DESTDIR)$(infodir) ; fi
+	if [ ! -d $(DESTDIR)$(infodir) ] ; then $(INSTALL_DIR) $(DESTDIR)$(infodir) ; fi
 	$(INSTALL_DATA) $(VPATH)/doc/$(pkgname).info $(DESTDIR)$(infodir)/$(pkgname).info
 	-install-info --info-dir=$(DESTDIR)$(infodir) $(DESTDIR)$(infodir)/$(pkgname).info
 
 install-man :
-	if test ! -d $(DESTDIR)$(mandir)/man1 ; then $(INSTALL_DIR) $(DESTDIR)$(mandir)/man1 ; fi
+	if [ ! -d $(DESTDIR)$(mandir)/man1 ] ; then $(INSTALL_DIR) $(DESTDIR)$(mandir)/man1 ; fi
 	$(INSTALL_DATA) $(VPATH)/doc/$(progname).1 $(DESTDIR)$(mandir)/man1/$(progname).1
 
 install-strip : all
@@ -145,6 +146,7 @@ dist :
 clean :
 	-rm -f $(progname) $(progname)_profiled $(objs) $(lib_objs) *.a
 	-rm -f $(progname)_shared $(sh_lib_objs) *.so.$(pkgversion)
+	-rm -f lzcheck.o
 
 distclean : clean
 	-rm -f Makefile config.status *.tar *.tar.lz

NEWS

@@ -1,4 +1,3 @@
-Changes in version 0.6:
+Changes in version 0.7:
 
-The LZ_compress_sync_flush mechanism has been fixed. (Last few bytes
+Compression time has been reduced by 4%.
-weren't inmediately available).

configure

@@ -5,13 +5,12 @@
 # This configure script is free software: you have unlimited permission
 # to copy, distribute and modify it.
 #
-# Date of this version: 2009-09-02
+# Date of this version: 2009-10-20
 
-invocation_name=$0
 args=
 no_create=
 pkgname=lzlib
-pkgversion=0.6
+pkgversion=0.7
 soversion=0
 progname=minilzip
 progname_shared=${progname}_shared
@@ -129,7 +128,7 @@ if [ x"${srcdir}" = x ] ; then
 	if [ ! -r ${srcdir}/${srctrigger} ] ; then srcdir=.. ; fi
 	if [ ! -r ${srcdir}/${srctrigger} ] ; then
 		## the sed command below emulates the dirname command
-		srcdir=`echo ${invocation_name} | sed -e 's,[^/]*$,,;s,/$,,;s,^$,.,'`
+		srcdir=`echo $0 | sed -e 's,[^/]*$,,;s,/$,,;s,^$,.,'`
 	fi
 fi
 
@@ -178,7 +177,7 @@ if [ x${no_create} = x ] ; then
 # This script is free software: you have unlimited permission
 # to copy, distribute and modify it.
 
-exec /bin/sh ${invocation_name} ${args} --no-create
+exec /bin/sh $0 ${args} --no-create
 EOF
 	chmod +x config.status
 fi

doc/lzlib.info

@@ -9,10 +9,10 @@ END-INFO-DIR-ENTRY
 
 File: lzlib.info,  Node: Top,  Next: Introduction,  Up: (dir)
 
-Lzlib
+Lzlib Manual
-*****
+************
 
-This manual is for Lzlib (version 0.6, 2 September 2009).
+This manual is for Lzlib (version 0.7, 20 October 2009).
 
 * Menu:
 
@@ -519,15 +519,15 @@ Concept Index
 
 Tag Table:
 Node: Top219
-Node: Introduction1015
+Node: Introduction1028
-Node: Library Version2790
+Node: Library Version2803
-Node: Buffering3435
+Node: Buffering3448
-Node: Compression Functions4542
+Node: Compression Functions4555
-Node: Decompression Functions10044
+Node: Decompression Functions10057
-Node: Error Codes13482
+Node: Error Codes13495
-Node: Data Format15418
+Node: Data Format15431
-Node: Examples17385
+Node: Examples17398
-Node: Problems18819
+Node: Problems18832
-Node: Concept Index19389
+Node: Concept Index19402
 
 End Tag Table

doc/lzlib.texinfo

@@ -1,12 +1,12 @@
 \input texinfo @c -*-texinfo-*-
 @c %**start of header
 @setfilename lzlib.info
-@settitle Lzlib
+@settitle Lzlib Manual
 @finalout
 @c %**end of header
 
-@set UPDATED 2 September 2009
+@set UPDATED 20 October 2009
-@set VERSION 0.6
+@set VERSION 0.7
 
 @dircategory Data Compression
 @direntry

encoder.cc

@@ -309,6 +309,7 @@ int LZ_encoder::best_pair_sequence( const int reps[num_rep_distances],
   trials[0].state = state;
   for( int i = 0; i < num_rep_distances; ++i ) trials[0].reps[i] = reps[i];
 
+  const uint8_t prev_byte = matchfinder[-1];
   const uint8_t cur_byte = matchfinder[0];
   const uint8_t match_byte = matchfinder[-reps[0]-1];
   unsigned int position = matchfinder.data_position();
@@ -318,9 +319,9 @@ int LZ_encoder::best_pair_sequence( const int reps[num_rep_distances],
   trials[1].prev_index = 0;
   trials[1].price = price0( bm_match[state()][pos_state] );
   if( state.is_char() )
-    trials[1].price += literal_encoder.price_symbol( matchfinder[-1], cur_byte );
+    trials[1].price += literal_encoder.price_symbol( prev_byte, cur_byte );
   else
-    trials[1].price += literal_encoder.price_matched( matchfinder[-1], cur_byte, match_byte );
+    trials[1].price += literal_encoder.price_matched( prev_byte, cur_byte, match_byte );
 
   const int match_price = price1( bm_match[state()][pos_state] );
   const int rep_match_price = match_price + price1( bm_rep[state()] );
@@ -354,9 +355,13 @@ int LZ_encoder::best_pair_sequence( const int reps[num_rep_distances],
   }
 
   for( int rep = 0; rep < num_rep_distances; ++rep )
+    {
+    const int price = rep_match_price +
+                      price_rep( rep, state, pos_state );
     for( int len = min_match_len; len <= replens[rep]; ++len )
-      trials[len].update( rep, 0, rep_match_price +
+      trials[len].update( rep, 0, price +
-                                  price_rep( rep, len, state, pos_state ) );
+                                  rep_match_len_encoder.price( len, pos_state ) );
+    }
 
   int cur = 0;
   int num_trials = main_len;
@@ -396,14 +401,15 @@ int LZ_encoder::best_pair_sequence( const int reps[num_rep_distances],
       mtf_reps( cur_trial.dis, cur_trial.reps );
       }
 
+    const uint8_t prev_byte = matchfinder[-1];
     const uint8_t cur_byte = matchfinder[0];
     const uint8_t match_byte = matchfinder[-cur_trial.reps[0]-1];
     const int pos_state = ++position & pos_state_mask;
     int next_price = cur_trial.price + price0( bm_match[cur_trial.state()][pos_state] );
     if( cur_trial.state.is_char() )
-      next_price += literal_encoder.price_symbol( matchfinder[-1], cur_byte );
+      next_price += literal_encoder.price_symbol( prev_byte, cur_byte );
     else
-      next_price += literal_encoder.price_matched( matchfinder[-1], cur_byte, match_byte );
+      next_price += literal_encoder.price_matched( prev_byte, cur_byte, match_byte );
     if( !matchfinder.move_pos() ) return 0;
 
     Trial & next_trial = trials[cur+1];
@@ -429,11 +435,13 @@ int LZ_encoder::best_pair_sequence( const int reps[num_rep_distances],
       while( len < len_limit && data[len] == data[len-dis] ) ++len;
       if( len >= min_match_len )
         {
+        const int price = rep_match_price +
+                          price_rep( rep, cur_trial.state, pos_state );
         while( num_trials < cur + len )
           trials[++num_trials].price = infinite_price;
         for( ; len >= min_match_len; --len )
-          trials[cur+len].update( rep, cur, rep_match_price +
+          trials[cur+len].update( rep, cur, price +
-                                  price_rep( rep, len, cur_trial.state, pos_state ) );
+                                  rep_match_len_encoder.price( len, pos_state ) );
         }
       }
 
@@ -447,7 +455,7 @@ int LZ_encoder::best_pair_sequence( const int reps[num_rep_distances],
       while( num_trials < cur + newlen )
         trials[++num_trials].price = infinite_price;
 
-      for( int len = newlen; len >= min_match_len; --len )
+      for( int len = min_match_len; len <= newlen; ++len )
         trials[cur+len].update( match_distances[len] + num_rep_distances, cur,
                                 normal_match_price +
                                 price_pair( match_distances[len], len, pos_state ) );

encoder.h

@@ -55,6 +55,7 @@ public:
 
 extern const Dis_slots dis_slots;
 
+
 class Prob_prices
   {
   int data[bit_model_total >> 2];
@@ -79,6 +80,7 @@ public:
 
 extern const Prob_prices prob_prices;
 
+
 inline int price0( const Bit_model & bm ) throw()
   { return prob_prices[bm.probability]; }
 
@@ -88,6 +90,7 @@ inline int price1( const Bit_model & bm ) throw()
 inline int price_bit( const Bit_model & bm, const int bit ) throw()
   { if( bit ) return price1( bm ); else return price0( bm ); }
 
+
 inline int price_symbol( const Bit_model bm[], int symbol, const int num_bits ) throw()
   {
   symbol |= ( 1 << num_bits );
@@ -101,6 +104,7 @@ inline int price_symbol( const Bit_model bm[], int symbol, const int num_bits )
   return price;
   }
 
+
 inline int price_symbol_reversed( const Bit_model bm[], int symbol,
                                   const int num_bits ) throw()
   {
@@ -116,6 +120,7 @@ inline int price_symbol_reversed( const Bit_model bm[], int symbol,
   return price;
   }
 
+
 inline int price_matched( const Bit_model bm[], const int symbol,
                           const int match_byte ) throw()
   {
@@ -424,7 +429,7 @@ class LZ_encoder
     {
     State state;
     int dis;
-    int prev_index;
+    int prev_index;	// index of prev trial in trials[]
     int price;		// dual use var; cumulative price, match length
     int reps[num_rep_distances];
     void update( const int d, const int p_i, const int pr ) throw()
@@ -489,25 +494,18 @@ class LZ_encoder
     return price0( bm_rep0[state()] ) + price0( bm_len[state()][pos_state] );
     }
 
-  int price_rep( const int rep, const int len, const State & state,
+  int price_rep( const int rep, const State & state,
                  const int pos_state ) const throw()
     {
-    int price = rep_match_len_encoder.price( len, pos_state );
+    if( rep == 0 ) return price0( bm_rep0[state()] ) +
-    if( rep == 0 )
+                          price1( bm_len[state()][pos_state] );
-      {
+    int price = price1( bm_rep0[state()] );
-      price += price0( bm_rep0[state()] );
+    if( rep == 1 )
-      price += price1( bm_len[state()][pos_state] );
+      price += price0( bm_rep1[state()] );
-      }
     else
       {
-      price += price1( bm_rep0[state()] );
+      price += price1( bm_rep1[state()] );
-      if( rep == 1 )
+      price += price_bit( bm_rep2[state()], rep - 2 );
-        price += price0( bm_rep1[state()] );
-      else
-        {
-        price += price1( bm_rep1[state()] );
-        price += price_bit( bm_rep2[state()], rep - 2 );
-        }
       }
     return price;
     }

lzcheck.cc

@@ -29,6 +29,11 @@
 #define ULLONG_MAX 0xFFFFFFFFFFFFFFFFULL
 #endif
 
+const int buffer_size = 65536;
+uint8_t in_buffer[buffer_size];
+uint8_t mid_buffer[buffer_size];
+uint8_t out_buffer[buffer_size];
+
 
 int main( const int argc, const char * argv[] )
   {
@@ -38,28 +43,15 @@ int main( const int argc, const char * argv[] )
     return 1;
     }
 
-  FILE *f = std::fopen( argv[1], "rb" );
+  FILE *file = std::fopen( argv[1], "rb" );
-  if( !f )
+  if( !file )
     {
     std::fprintf( stderr, "Can't open file `%s' for reading\n", argv[1] );
     return 1;
     }
+//  std::fprintf( stderr, "lzcheck: testing file `%s'\n", argv[1] );
 
-  const int in_buffer_size = 1 << 20;
+  const int dictionary_size = 1 << 20;
-  const int mid_buffer_size = 65536;
-  const int out_buffer_size = in_buffer_size;
-  uint8_t in_buffer[in_buffer_size];
-  uint8_t mid_buffer[mid_buffer_size];
-  uint8_t out_buffer[out_buffer_size];
-  const int in_size = std::fread( in_buffer, 1, in_buffer_size, f );
-  if( in_size >= in_buffer_size )
-    {
-    std::fprintf( stderr, "input file `%s' too big.\n", argv[1] );
-    return 1;
-    }
-  std::fclose( f );
-
-  const int dictionary_size = in_buffer_size;
   const int match_len_limit = 80;
   const long long member_size = LLONG_MAX;
   void * encoder = LZ_compress_open( dictionary_size, match_len_limit,
@@ -85,27 +77,35 @@ int main( const int argc, const char * argv[] )
     return 1;
     }
 
-  for( int l = 0, r = 0; r < in_size; l = r )
+  while( true )
     {
-    while( ++r < in_size && in_buffer[r-1] != '\n' ) ;
+    const int read_size = std::fread( in_buffer, 1, buffer_size, file );
-    LZ_compress_write( encoder, in_buffer + l, r - l );
+    if( read_size <= 0 ) break;
-    LZ_compress_sync_flush( encoder );
-    int mid_size = LZ_compress_read( encoder, mid_buffer, mid_buffer_size );
-    LZ_decompress_write( decoder, mid_buffer, mid_size );
-    int out_size = LZ_decompress_read( decoder, out_buffer, out_buffer_size );
 
-    if( out_size != r - l || std::memcmp( in_buffer + l, out_buffer, out_size ) )
+    for( int l = 0, r = 1; r <= read_size; l = r, ++r )
       {
-      std::printf( "sync error at pos %d. in_size = %d, out_size = %d\n",
+      while( r < read_size && in_buffer[r-1] != '\n' ) ++r;
-                   l, r - l, out_size );
+      const int in_size = LZ_compress_write( encoder, in_buffer + l, r - l );
-      for( int i = l; i < r; ++i ) std::putchar( in_buffer[i] );
+      if( in_size < r - l ) r = l + in_size;
-      if( in_buffer[r-1] != '\n' ) std::putchar( '\n' );
+      LZ_compress_sync_flush( encoder );
-      for( int i = 0; i < out_size; ++i ) std::putchar( out_buffer[i] );
+      const int mid_size = LZ_compress_read( encoder, mid_buffer, buffer_size );
-      std::putchar( '\n' );
+      LZ_decompress_write( decoder, mid_buffer, mid_size );
+      const int out_size = LZ_decompress_read( decoder, out_buffer, buffer_size );
+
+      if( out_size != in_size || std::memcmp( in_buffer + l, out_buffer, out_size ) )
+        {
+        std::printf( "sync error at pos %d. in_size = %d, out_size = %d\n",
+                     l, in_size, out_size );
+        for( int i = 0; i < in_size; ++i ) std::putchar( in_buffer[l+i] );
+        if( in_buffer[l+in_size-1] != '\n' ) std::putchar( '\n' );
+        for( int i = 0; i < out_size; ++i ) std::putchar( out_buffer[i] );
+        std::putchar( '\n' );
+        }
       }
     }
 
   LZ_decompress_close( decoder );
   LZ_compress_close( encoder );
+  std::fclose( file );
   return 0;
   }

lzlib.h

@@ -29,7 +29,7 @@
 extern "C" {
 #endif
 
-const char * const LZ_version_string = "0.6";
+const char * const LZ_version_string = "0.7";
 
 enum { min_dictionary_bits = 12,
        min_dictionary_size = 1 << min_dictionary_bits,

main.cc

@@ -183,7 +183,7 @@ const char * format_num( long long num, long long limit = 9999,
   }
 
 
-long long getnum( const char * ptr, const int bs,
+long long getnum( const char * ptr, const int bs = 0,
                   const long long llimit = LLONG_MIN + 1,
                   const long long ulimit = LLONG_MAX ) throw()
   {
@@ -462,8 +462,6 @@ int compress( const long long member_size, const long long volume_size,
       if( in_size == 0 ) LZ_compress_finish( encoder );
       else if( in_size != LZ_compress_write( encoder, in_buffer, in_size ) )
         internal_error( "library error" );
-//      for( int i = 0; i < 10000; ++i )
-//        LZ_compress_sync_flush( encoder );
       }
     int out_size = LZ_compress_read( encoder, out_buffer, out_buffer_size );
 //    std::fprintf( stderr, "%6d in_size, %5d out_size.\n", in_size, out_size );
@@ -601,15 +599,15 @@ int decompress( const int inhandle, const Pretty_print & pp,
 extern "C" void signal_handler( int ) throw()
   {
   show_error( "Control-C or similar caught, quitting." );
-  cleanup_and_fail( 0 );
+  cleanup_and_fail( 1 );
   }
 
 
 void set_signals() throw()
   {
-  signal( SIGTERM, signal_handler );
   signal( SIGHUP, signal_handler );
   signal( SIGINT, signal_handler );
+  signal( SIGTERM, signal_handler );
   }
 
 } // end namespace
@@ -838,7 +836,7 @@ int main( const int argc, const char * argv[] )
       const int eindex = extension_index( input_filename );
       inhandle = open_instream( input_filename, &in_stats, program_mode,
                                 eindex, force, to_stdout );
-      if( inhandle < 0 ) continue;
+      if( inhandle < 0 ) { if( retval < 1 ) retval = 1; continue; }
       if( program_mode != m_test )
         {
         if( to_stdout ) outhandle = STDOUT_FILENO;

testsuite/check.sh

@@ -18,7 +18,7 @@ if [ ! -x "${LZIP}" ] ; then
 	exit 1
 fi
 
-if [ -d tmp ] ; then rm -r tmp ; fi
+if [ -d tmp ] ; then rm -rf tmp ; fi
 mkdir tmp
 echo -n "testing minilzip..."
 cd "${objdir}"/tmp
@@ -29,7 +29,7 @@ fail=0
 "${LZIP}" -cd "${testdir}"/COPYING.lz > copy || fail=1
 cmp in copy || fail=1
 
-for i in s4096 1 2 3 4 5 6 7 8 9; do
+for i in s4096 1 2 3 4 5 6 7 8; do
 	"${LZIP}" -k -$i in || fail=1
 	mv -f in.lz copy.lz || fail=1
 	echo -n "garbage" >> copy.lz || fail=1
@@ -38,7 +38,7 @@ for i in s4096 1 2 3 4 5 6 7 8 9; do
 	echo -n .
 done
 
-for i in s4096 1 2 3 4 5 6 7 8 9; do
+for i in s4096 1 2 3 4 5 6 7 8; do
 	"${LZIP}" -c -$i in > out || fail=1
 	echo -n "g" >> out || fail=1
 	"${LZIP}" -cd out > copy || fail=1
@@ -46,25 +46,25 @@ for i in s4096 1 2 3 4 5 6 7 8 9; do
 	echo -n .
 done
 
-for i in s4096 1 2 3 4 5 6 7 8 9; do
+for i in s4096 1 2 3 4 5 6 7 8; do
 	"${LZIP}" -c -$i < in > out || fail=1
 	"${LZIP}" -d < out > copy || fail=1
 	cmp in copy || fail=1
 	echo -n .
 done
 
-for i in s4096 1 2 3 4 5 6 7 8 9; do
+for i in s4096 1 2 3 4 5 6 7 8; do
 	"${LZIP}" -f -$i -o out < in || fail=1
 	"${LZIP}" -df -o copy < out.lz || fail=1
 	cmp in copy || fail=1
 	echo -n .
 done
 
-"${LZCHECK}" in || fail=1
+"${LZCHECK}" in 2>/dev/null || fail=1
 echo -n .
 
 echo
-if [ ${fail} = 0 ]; then
+if [ ${fail} = 0 ] ; then
 	echo "tests completed successfully."
 	cd "${objdir}" && rm -r tmp
 else