Adding upstream version 1.25~pre1.

Signed-off-by: Daniel Baumann <daniel@debian.org>

COPYING

@@ -1,8 +1,7 @@
                     GNU GENERAL PUBLIC LICENSE
                        Version 2, June 1991
 
- Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc. <http://fsf.org/>
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 

ChangeLog

@@ -1,3 +1,15 @@
+2024-10-01  Antonio Diaz Diaz  <antonio@gnu.org>
+
+	* Version 1.25-pre1 released.
+	* New options '-F, --fec', '-0' to '-9', '-b, --block-size',
+	  '--fec-file', '-r, --recursive', and '-R, --dereference-recursive'.
+	* Change short name of option '--byte-repair' to '-B'.
+	* New options '--ignore-empty' and '--ignore-nonzero'.
+	* Rename option '--clear-marking' to '--nonzero-repair'.
+	* Remove options '--empty-error' and '--marking-error'.
+	* Remove decompression support for Sync Flush marker.
+	* testsuite: Require lzip/clzip. Add fox6_nz.lz. Remove fox6_mark.lz.
+
 2024-01-20  Antonio Diaz Diaz  <antonio@gnu.org>
 
 	* Version 1.24 released.
@@ -122,8 +134,8 @@
 	* repair.cc: Repair a damaged dictionary size in the header.
 	* repair.cc: Try bytes at offsets 7 to 11 first.
 	* Decompression time has been reduced by 2%.
-	* main.cc (decompress): Print up to 6 bytes of trailing data
+	* main.cc (decompress): Print up to 6 bytes of trailing data when
-	  when '-tvvvv' is specified.
+	  '-tvvvv' is specified.
 	* decoder.cc (verify_trailer): Remove test of final code.
 	* main.cc (main): Delete '--output' file if infd is a terminal.
 	* main.cc (main): Don't use stdin more than once.
@@ -166,8 +178,8 @@
 2013-09-14  Antonio Diaz Diaz  <antonio@gnu.org>
 
 	* Version 1.15 released.
-	* repair.cc: Repair multimember files with up to one byte error
+	* repair.cc: Repair multimember files with up to one byte error per
-	  per member.
+	  member.
 	* merge.cc: Merge multimember files.
 	* main.cc (show_header): Don't show header version.
 	* lziprecover.texinfo: New chapters 'Repairing files',
@@ -189,13 +201,13 @@
 2012-02-24  Antonio Diaz Diaz  <ant_diaz@teleline.es>
 
 	* Version 1.13 released.
-	* Lziprecover is now distributed in its own package. Until
+	* Lziprecover is now distributed in its own package. Until version
-	  version 1.12 it was included in the lzip package. Previous
+	  1.12 it was included in the lzip package. Previous entries in this
-	  entries in this file are taken from there.
+	  file are taken from there.
 	* lziprecover.cc: Rename to main.cc.
 	* New files merge.cc, repair.cc, split.cc, and range_dec.cc.
-	* main.cc: Add decompressor options (-c, -d, -k, -t) so that
+	* main.cc: Add decompressor options (-c, -d, -k, -t) so that an
-	  an external decompressor is not needed for recovery nor for
+	  external decompressor is not needed for recovery nor for
 	  "make check".
 	* New option '-D, --range-decompress', which extracts a range of
 	  bytes decompressing only the members containing the desired data.
@@ -226,8 +238,8 @@
 	  This change also prevents (harmless) access to uninitialized
 	  memory when decompressing a corrupt file.
 	* lziprecover.cc: New options '-f, --force' and '-o, --output'.
-	* lziprecover.cc: New option '-s, --split' to select the until
+	* lziprecover.cc: New option '-s, --split' to select the until now
-	  now only operation of splitting multimember files.
+	  only operation of splitting multimember files.
 	* lziprecover.cc: If no operation is specified, warn the user and do
 	  nothing.
 
@@ -246,6 +258,5 @@
 
 Copyright (C) 2009-2024 Antonio Diaz Diaz.
 
-This file is a collection of facts, and thus it is not copyrightable,
+This file is a collection of facts, and thus it is not copyrightable, but just
-but just in case, you have unlimited permission to copy, distribute, and
+in case, you have unlimited permission to copy, distribute, and modify it.
-modify it.

Makefile.in

@@ -8,8 +8,9 @@ SHELL = /bin/sh
 CAN_RUN_INSTALLINFO = $(SHELL) -c "install-info --version" > /dev/null 2>&1
 
 objs = arg_parser.o alone_to_lz.o lzip_index.o list.o byte_repair.o \
-       dump_remove.o lunzcrash.o md5.o merge.o mtester.o nrep_stats.o \
+       dump_remove.o fec_create.o fec_repair.o gf8.o gf16.o lunzcrash.o \
-       range_dec.o reproduce.o split.o decoder.o main.o
+       md5.o merge.o mtester.o nrep_stats.o range_dec.o recursive.o \
+       reproduce.o split.o decoder.o main.o
 unzobjs = arg_parser.o unzcrash.o
 
 
@@ -22,7 +23,7 @@ unzobjs = arg_parser.o unzcrash.o
 all : $(progname)
 
 $(progname) : $(objs)
-	$(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $(objs)
+	$(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $(objs) $(LIBS)
 
 unzcrash : $(unzobjs)
 	$(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $(unzobjs)
@@ -38,7 +39,8 @@ unzcrash.o : unzcrash.cc
 
 # prevent 'make' from trying to remake source files
 $(VPATH)/configure $(VPATH)/Makefile.in $(VPATH)/doc/$(pkgname).texi : ;
-%.h %.cc : ;
+MAKEFLAGS += -r
+.SUFFIXES :
 
 $(objs)       : Makefile
 alone_to_lz.o : lzip.h common.h mtester.h
@@ -46,15 +48,20 @@ arg_parser.o  : arg_parser.h
 byte_repair.o : lzip.h common.h mtester.h lzip_index.h
 decoder.o     : lzip.h common.h decoder.h
 dump_remove.o : lzip.h common.h lzip_index.h
+fec_create.o  : lzip.h common.h md5.h fec.h
+fec_repair.o  : lzip.h common.h md5.h fec.h
+gf8.o         : lzip.h common.h md5.h fec.h
+gf16.o        : lzip.h common.h md5.h fec.h
 list.o        : lzip.h common.h lzip_index.h
 lunzcrash.o   : lzip.h common.h md5.h mtester.h lzip_index.h
 lzip_index.o  : lzip.h common.h lzip_index.h
-main.o        : arg_parser.h lzip.h common.h decoder.h main_common.cc
+main.o      : arg_parser.h lzip.h common.h decoder.h md5.h fec.h main_common.cc
 md5.o         : md5.h
 merge.o       : lzip.h common.h decoder.h lzip_index.h
 mtester.o     : lzip.h common.h md5.h mtester.h
 nrep_stats.o  : lzip.h common.h lzip_index.h
 range_dec.o   : lzip.h common.h decoder.h lzip_index.h
+recursive.o   : lzip.h common.h md5.h fec.h
 reproduce.o   : lzip.h common.h md5.h mtester.h lzip_index.h
 split.o       : lzip.h common.h lzip_index.h
 unzcrash.o    : Makefile arg_parser.h common.h main_common.cc
@@ -141,21 +148,23 @@ dist : doc
 	  $(DISTNAME)/testsuite/check.sh \
 	  $(DISTNAME)/testsuite/fox6_bad1.txt \
 	  $(DISTNAME)/testsuite/test.txt \
-	  $(DISTNAME)/testsuite/test21723.txt \
+	  $(DISTNAME)/testsuite/test21636.txt \
 	  $(DISTNAME)/testsuite/test_bad[6-9].txt \
 	  $(DISTNAME)/testsuite/test_3m.txt.lz.md5 \
 	  $(DISTNAME)/testsuite/fox.lz \
 	  $(DISTNAME)/testsuite/fox_*.lz \
 	  $(DISTNAME)/testsuite/fox6.lz \
+	  $(DISTNAME)/testsuite/fox6_nz.lz \
 	  $(DISTNAME)/testsuite/fox6_sc[1-6].lz \
 	  $(DISTNAME)/testsuite/fox6_bad[1-6].lz \
-	  $(DISTNAME)/testsuite/fox6_mark.lz \
 	  $(DISTNAME)/testsuite/numbers.lz \
 	  $(DISTNAME)/testsuite/numbersbt.lz \
 	  $(DISTNAME)/testsuite/test.txt.lz \
 	  $(DISTNAME)/testsuite/test.txt.lzma \
 	  $(DISTNAME)/testsuite/test_bad[1-9].lz \
-	  $(DISTNAME)/testsuite/test_em.txt.lz
+	  $(DISTNAME)/testsuite/test_em.txt.lz \
+	  $(DISTNAME)/testsuite/test.txt.lz.fec \
+	  $(DISTNAME)/testsuite/test.txt.lz.fec16
 	rm -f $(DISTNAME)
 	lzip -v -9 $(DISTNAME).tar
 

NEWS

@@ -1,35 +1,31 @@
-Changes in version 1.24:
+Changes in version 1.25:
 
-The option '--empty-error', which forces exit status 2 if any empty member
+The option '-F, --fec', which implements Forward Error Correction (FEC), has
-is found, has been added.
+been added.
 
-The option '--marking-error', which forces exit status 2 if the first LZMA
+The options '-0' to '-9' (FEC fragmentation level) have been added.
-byte is non-zero in any member, has been added.
 
-The option '--clear-marking', which sets to zero the first LZMA byte of each
+The option '-b, --block-size', which sets the FEC block size, has been added.
-member, has been added.
 
-The keyword 'empty' is now recognized in the argument of '--dump',
+The option '--fec-file', which sets the fec file to be used, has been added.
-'--remove', and '--strip'.
 
-The option '--repair' has been renamed to '--byte-repair'.
+The options '-r, --recursive' and '-R, --dereference-recursive' have been
+added for recursive creation and reading of fec files.
 
-The option '--debug-repair' has been renamed to '--debug-byte-repair'.
+The short name of option '--byte-repair' has been changed to "-B".
 
-File diagnostics have been reformatted as 'PROGRAM: FILE: MESSAGE'.
+The option '--ignore-empty', which makes lziprecover ignore empty members in
+multimember files when decompressing, testing, or listing, has been added.
+By default lziprecover now exits with error status 2 if any empty member is
+found in a multimember file.
 
-Diagnostics caused by invalid arguments to command-line options now show the
+The option '--ignore-nonzero', which makes lziprecover ignore a nonzero
-argument and the name of the option.
+first byte in the LZMA stream when decompressing or testing, has been added.
+By default lziprecover now exits with error status 2 if the first LZMA byte
+is nonzero in any member of the input files.
 
-The option '-o, --output' now preserves dates, permissions, and ownership of
+The option '--clear-marking' has been renamed to '--nonzero-repair'.
-the file, when decompressing exactly one file.
 
-The option '-o, --output' now creates missing intermediate directories when
+Options '--empty-error' and '--marking-error' have been removed.
-writing to a file.
 
-The option '--no-verify' of unzcrash has been renamed to '--no-check'.
+Lzip 1.16 (or clzip 1.6) or newer is required to run the tests.
-
-The variable MAKEINFO has been added to configure and Makefile.in.
-
-The makefile target 'install-as-lzip' has been removed because '--reproduce'
-needs a lzip compressor (not just a decompressor) named 'lzip' by default.

README

@@ -1,11 +1,8 @@
 Description
 
 Lziprecover is a data recovery tool and decompressor for files in the lzip
-compressed data format (.lz). Lziprecover is able to repair slightly damaged
+compressed data format (.lz). Lziprecover also provides Forward Error
-files (up to one single-byte error per member), produce a correct file by
+Correction (FEC) able to repair any kind of file.
-merging the good parts of two or more damaged copies, reproduce a missing
-(zeroed) sector using a reference file, extract data from damaged files,
-decompress files, and test integrity of files.
 
 Lziprecover can remove the damaged members from multimember files, for
 example multimember tar.lz archives.
@@ -13,9 +10,6 @@ example multimember tar.lz archives.
 Lziprecover provides random access to the data in multimember files; it only
 decompresses the members containing the desired data.
 
-Lziprecover facilitates the management of metadata stored as trailing data
-in lzip files.
-
 Lziprecover is not a replacement for regular backups, but a last line of
 defense for the case where the backups are also damaged.
 
@@ -59,8 +53,8 @@ GNU ddrescue + lziprecover is the recommended option for recovering data
 from damaged lzip files.
 
 If a file is too damaged for lziprecover to repair it, all the recoverable
-data in all members of the file can be extracted in one step with the
+data in all members of the file can be extracted with the command
-command 'lziprecover -cd --ignore-errors file.lz > file'.
+'lziprecover -cd --ignore-errors file.lz > file'.
 
 When recovering data, lziprecover takes as arguments the names of the
 damaged files and writes zero or more recovered files depending on the
@@ -70,14 +64,6 @@ files themselves are kept unchanged.
 When decompressing or testing file integrity, lziprecover behaves like lzip
 or lunzip.
 
-To give you an idea of its possibilities, when merging two copies, each of
-them with one damaged area affecting 1 percent of the copy, the probability
-of obtaining a correct file is about 98 percent. With three such copies the
-probability rises to 99.97 percent. For large files (a few MB) with small
-errors (one sector damaged per copy), the probability approaches 100 percent
-even with only two copies. (Supposing that the errors are randomly located
-inside each copy).
-
 The lziprecover package also includes unzcrash, a program written to test
 robustness to decompression of corrupted data, inspired by unzcrash.c from
 Julian Seward's bzip2. Type 'make unzcrash' in the lziprecover source

alone_to_lz.cc

@@ -50,7 +50,7 @@ uint8_t * read_file( const int infd, long * const file_sizep,
   while( file_size >= buffer_size - 20 && !errno )
     {
     if( buffer_size >= LONG_MAX )
-      { show_file_error( filename, "Input file is larger than LONG_MAX." );
+      { show_file_error( filename, large_file_msg );
         std::free( buffer ); return 0; }
     buffer_size = ( buffer_size <= LONG_MAX / 2 ) ? 2 * buffer_size : LONG_MAX;
     uint8_t * const tmp = (uint8_t *)std::realloc( buffer, buffer_size );
@@ -61,7 +61,7 @@ uint8_t * read_file( const int infd, long * const file_sizep,
     }
   if( errno )
     {
-    show_file_error( filename, "Error reading input file", errno );
+    show_file_error( filename, read_error_msg, errno );
     std::free( buffer ); return 0;
     }
   *file_sizep = file_size;
@@ -88,7 +88,7 @@ int alone_to_lz( const int infd, const Pretty_print & pp )
   uint8_t * const buffer = read_file( infd, &file_size, pp.name() );
   if( !buffer ) return 1;
   if( file_size < lzma_header_size )
-    { show_file_error( pp.name(), "Input file is too short." );
+    { show_file_error( pp.name(), short_file_msg );
       std::free( buffer ); return 2; }
 
   if( buffer[0] != 93 )			// (45 * 2) + (9 * 0) + 3
@@ -100,7 +100,7 @@ int alone_to_lz( const int infd, const Pretty_print & pp )
       show_file_error( pp.name(), "Input file has non-default LZMA properties." );
     std::free( buffer ); return 2;
     }
-  for( int i = 5; i < 13; ++i ) if( buffer[i] != 0xFF )
+  for( int i = 5; i < lzma_header_size; ++i ) if( buffer[i] != 0xFF )
     { show_file_error( pp.name(), "Input file is non-streamed." );
       std::free( buffer ); return 2; }
 
@@ -113,10 +113,12 @@ int alone_to_lz( const int infd, const Pretty_print & pp )
   Lzip_header & header = *(Lzip_header *)( buffer + offset );
   header.set_magic();
   header.dictionary_size( dictionary_size );
+  buffer[lzma_header_size] = 0;			// reset first LZMA byte
   for( int i = 0; i < Lzip_trailer::size; ++i ) buffer[file_size++] = 0;
+  const long lzip_size = file_size - offset;
   // compute and fill trailer
   {
-  LZ_mtester mtester( buffer + offset, file_size - offset, dictionary_size );
+  LZ_mtester mtester( buffer + offset, lzip_size, dictionary_size );
   const int result = mtester.test_member();
   if( result == 1 && orig_dictionary_size > max_dictionary_size )
     { pp( "dictionary size is too large" ); std::free( buffer ); return 2; }
@@ -136,10 +138,10 @@ int alone_to_lz( const int infd, const Pretty_print & pp )
   trailer.member_size( mtester.member_position() );
   }
   // check converted member
-  LZ_mtester mtester( buffer + offset, file_size - offset, dictionary_size );
+  LZ_mtester mtester( buffer + offset, lzip_size, dictionary_size );
   if( mtester.test_member() != 0 || !mtester.finished() )
     { pp( "conversion failed" ); std::free( buffer ); return 2; }
-  if( writeblock( outfd, buffer + offset, file_size - offset ) != file_size - offset )
+  if( writeblock( outfd, buffer + offset, lzip_size ) != lzip_size )
     {
     show_error( "Error writing output file", errno );
     std::free( buffer ); return 1;

arg_parser.cc

@@ -75,19 +75,19 @@ bool Arg_parser::parse_long_option( const char * const opt, const char * const a
       error_ += "' requires an argument";
       return false;
       }
-    data.back().argument = &opt[len+3];
+    data.back().argument = &opt[len+3];		// argument may be empty
     return true;
     }
 
-  if( options[index].has_arg == yes )
+  if( options[index].has_arg == yes || options[index].has_arg == yme )
     {
-    if( !arg || !arg[0] )
+    if( !arg || ( options[index].has_arg == yes && !arg[0] ) )
       {
       error_ = "option '--"; error_ += options[index].long_name;
       error_ += "' requires an argument";
       return false;
       }
-    ++argind; data.back().argument = arg;
+    ++argind; data.back().argument = arg;	// argument may be empty
     return true;
     }
 
@@ -123,15 +123,16 @@ bool Arg_parser::parse_short_option( const char * const opt, const char * const
       {
       data.back().argument = &opt[cind]; ++argind; cind = 0;
       }
-    else if( options[index].has_arg == yes )
+    else if( options[index].has_arg == yes || options[index].has_arg == yme )
       {
-      if( !arg || !arg[0] )
+      if( !arg || ( options[index].has_arg == yes && !arg[0] ) )
         {
         error_ = "option requires an argument -- '"; error_ += c;
         error_ += '\'';
         return false;
         }
-      data.back().argument = arg; ++argind; cind = 0;
+      ++argind; cind = 0;
+      data.back().argument = arg;	// argument may be empty
       }
     }
   return true;

arg_parser.h

@@ -36,14 +36,18 @@
    The argument '--' terminates all options; any following arguments are
    treated as non-option arguments, even if they begin with a hyphen.
 
-   The syntax for optional option arguments is '-<short_option><argument>'
+   The syntax of options with an optional argument is
-   (without whitespace), or '--<long_option>=<argument>'.
+   '-<short_option><argument>' (without whitespace), or
+   '--<long_option>=<argument>'.
+
+   The syntax of options with an empty argument is '-<short_option> ""',
+   '--<long_option> ""', or '--<long_option>=""'.
 */
 
 class Arg_parser
   {
 public:
-  enum Has_arg { no, yes, maybe };
+  enum Has_arg { no, yes, maybe, yme };		// yme = yes but maybe empty
 
   struct Option
     {

byte_repair.cc

@@ -69,10 +69,10 @@ int repair_dictionary_size( uint8_t * const mbuffer, const long msize )
   const bool valid_ds = isvalid_ds( dictionary_size );
   if( valid_ds && dictionary_size >= data_size ) return 0;	// can't be bad
 
-  const unsigned long long dictionary_size_9 = 1 << 25;	// dict size of opt -9
+  const unsigned long long dict_size_9 = 1 << 25;	// dict size of opt -9
-  if( !valid_ds || dictionary_size < dictionary_size_9 )
+  if( !valid_ds || dictionary_size < dict_size_9 )
     {
-    dictionary_size = std::min( data_size, dictionary_size_9 );
+    dictionary_size = std::min( data_size, dict_size_9 );
     if( dictionary_size < min_dictionary_size )
       dictionary_size = min_dictionary_size;
     LZ_mtester mtester( mbuffer, msize, dictionary_size );
@@ -82,7 +82,7 @@ int repair_dictionary_size( uint8_t * const mbuffer, const long msize )
     if( result != 1 || mtester.max_distance() <= dictionary_size ||
         mtester.max_distance() > max_dictionary_size ) return 0;
     }
-  if( data_size > dictionary_size_9 )
+  if( data_size > dict_size_9 )
     {
     dictionary_size =
       std::min( data_size, (unsigned long long)max_dictionary_size );
@@ -174,7 +174,7 @@ uint8_t * read_member( const int infd, const long long mpos,
   uint8_t * const buffer = new uint8_t[msize];
 
   if( readblock( infd, buffer, msize ) != msize )
-    { show_file_error( filename, "Error reading input file", errno );
+    { show_file_error( filename, read_error_msg, errno );
       delete[] buffer; return 0; }
   return buffer;
   }
@@ -266,7 +266,8 @@ int byte_repair( const std::string & input_filename,
     }
   if( !close_outstream( &in_stats ) ) return 1;
   if( verbosity >= 1 )
-    std::fputs( "Copy of input file repaired successfully.\n", stdout );
+    std::printf( "Repaired copy of '%s' written to '%s'\n",
+                 filename, output_filename.c_str() );
   return 0;
   }
 
@@ -287,7 +288,8 @@ int debug_delay( const char * const input_filename,
   if( range.end() > lzip_index.cdata_size() )
     range.size( std::max( 0LL, lzip_index.cdata_size() - range.pos() ) );
   if( range.size() <= 0 )
-    { show_file_error( input_filename, "Nothing to do." ); return 0; }
+    { show_file_error( input_filename, "Nothing to do; range is empty." );
+      return 0; }
 
   for( long i = 0; i < lzip_index.members(); ++i )
     {
@@ -370,7 +372,8 @@ int debug_byte_repair( const char * const input_filename,
   for( ; idx < lzip_index.members(); ++idx )
     if( lzip_index.mblock( idx ).includes( bad_byte.pos ) ) break;
   if( idx >= lzip_index.members() )
-    { show_file_error( input_filename, "Nothing to do." ); return 0; }
+    { show_file_error( input_filename, "Nothing to do; byte is beyond EOF." );
+      return 0; }
 
   const long long mpos = lzip_index.mblock( idx ).pos();
   const long long msize = lzip_index.mblock( idx ).size();

common.h

@@ -38,9 +38,14 @@ struct Bad_byte
   };
 
 
+const char * const large_file_msg = "Input file is too large for this computer.";
 const char * const mem_msg = "Not enough memory.";
+const char * const read_error_msg = "Read error";
 
 // defined in main_common.cc
+extern int verbosity;
+
+const char * format_num3( long long num );
 void show_error( const char * const msg, const int errcode = 0,
                  const bool help = false );
 void show_file_error( const char * const filename, const char * const msg,

configure

@@ -6,7 +6,7 @@
 # to copy, distribute, and modify it.
 
 pkgname=lziprecover
-pkgversion=1.24
+pkgversion=1.25-pre1
 progname=lziprecover
 srctrigger=doc/${pkgname}.texi
 
@@ -24,6 +24,7 @@ CXX=g++
 CPPFLAGS=
 CXXFLAGS='-Wall -W -O2'
 LDFLAGS=
+LIBS=-lpthread
 MAKEINFO=makeinfo
 
 # checking whether we are using GNU C++.
@@ -70,6 +71,7 @@ while [ $# != 0 ] ; do
 		echo "  CXXFLAGS=OPTIONS      command-line options for the C++ compiler [${CXXFLAGS}]"
 		echo "  CXXFLAGS+=OPTIONS     append options to the current value of CXXFLAGS"
 		echo "  LDFLAGS=OPTIONS       command-line options for the linker [${LDFLAGS}]"
+		echo "  LIBS=OPTIONS          libraries to pass to the linker [${LIBS}]"
 		echo "  MAKEINFO=NAME         makeinfo program to use [${MAKEINFO}]"
 		echo
 		exit 0 ;;
@@ -98,6 +100,7 @@ while [ $# != 0 ] ; do
 	CXXFLAGS=*)  CXXFLAGS=${optarg} ;;
 	CXXFLAGS+=*) CXXFLAGS="${CXXFLAGS} ${optarg}" ;;
 	LDFLAGS=*)    LDFLAGS=${optarg} ;;
+	LIBS=*)          LIBS="${optarg} ${LIBS}" ;;
 	MAKEINFO=*)  MAKEINFO=${optarg} ;;
 
 	--*)
@@ -109,7 +112,7 @@ while [ $# != 0 ] ; do
 		exit 1 ;;
 	esac
 
-	# Check if the option took a separate argument
+	# Check whether the option took a separate argument
 	if [ "${arg2}" = yes ] ; then
 		if [ $# != 0 ] ; then args="${args} \"$1\"" ; shift
 		else echo "configure: Missing argument to '${option}'" 1>&2
@@ -167,6 +170,7 @@ echo "CXX = ${CXX}"
 echo "CPPFLAGS = ${CPPFLAGS}"
 echo "CXXFLAGS = ${CXXFLAGS}"
 echo "LDFLAGS = ${LDFLAGS}"
+echo "LIBS = ${LIBS}"
 echo "MAKEINFO = ${MAKEINFO}"
 rm -f Makefile
 cat > Makefile << EOF
@@ -191,6 +195,7 @@ CXX = ${CXX}
 CPPFLAGS = ${CPPFLAGS}
 CXXFLAGS = ${CXXFLAGS}
 LDFLAGS = ${LDFLAGS}
+LIBS = ${LIBS}
 MAKEINFO = ${MAKEINFO}
 EOF
 cat "${srcdir}/Makefile.in" >> Makefile

decoder.cc

@@ -76,7 +76,7 @@ bool Range_decoder::read_block()
   if( !at_stream_end )
     {
     stream_pos = readblock( infd, buffer, buffer_size );
-    if( stream_pos != buffer_size && errno ) throw Error( "Read error" );
+    if( stream_pos != buffer_size && errno ) throw Error( read_error_msg );
     at_stream_end = ( stream_pos < buffer_size );
     partial_member_pos += pos;
     pos = 0;
@@ -108,8 +108,7 @@ void LZ_decoder::flush_data()
   }
 
 
-int LZ_decoder::check_trailer( const Pretty_print & pp,
+bool LZ_decoder::check_trailer( const Pretty_print & pp ) const
-                               const bool ignore_empty ) const
   {
   Lzip_trailer trailer;
   int size = rdec.read_data( trailer.data, trailer.size );
@@ -154,8 +153,7 @@ int LZ_decoder::check_trailer( const Pretty_print & pp,
         std::fprintf( stderr, "Member size mismatch; stored %llu (0x%llX), computed %llu (0x%llX)\n",
                       tm_size, tm_size, member_size, member_size ); }
     }
-  if( error ) return 3;
+  if( error ) return false;
-  if( !ignore_empty && data_size == 0 ) return 5;
   if( verbosity >= 2 )
     {
     if( verbosity >= 4 ) show_header( dictionary_size );
@@ -175,15 +173,14 @@ int LZ_decoder::check_trailer( const Pretty_print & pp,
     pp();
     std::fprintf( stderr, "Range decoder final code is %08X\n", rdec.get_code() );
     }
-  return 0;
+  return true;
   }
 
 
 /* Return value: 0 = OK, 1 = decoder error, 2 = unexpected EOF,
                  3 = trailer error, 4 = unknown marker found,
-                 5 = empty member found, 6 = marked member found. */
+                 5 = nonzero first LZMA byte found. */
-int LZ_decoder::decode_member( const Cl_options & cl_opts,
+int LZ_decoder::decode_member( const Pretty_print & pp, const bool ignore_nonzero )
-                               const Pretty_print & pp )
   {
   Bit_model bm_literal[1<<literal_context_bits][0x300];
   Bit_model bm_match[State::states][pos_states];
@@ -203,7 +200,7 @@ int LZ_decoder::decode_member( const Cl_options & cl_opts,
   unsigned rep3 = 0;
   State state;
 
-  if( !rdec.load( cl_opts.ignore_marking ) ) return 6;
+  if( !rdec.load( ignore_nonzero ) ) return 5;
   while( !rdec.finished() )
     {
     const int pos_state = data_position() & pos_state_mask;
@@ -267,14 +264,9 @@ int LZ_decoder::decode_member( const Cl_options & cl_opts,
             rdec.normalize();
             flush_data();
             if( len == min_match_len )		// End Of Stream marker
-              return check_trailer( pp, cl_opts.ignore_empty );
+              { if( check_trailer( pp ) ) return 0; else return 3; }
-            if( len == min_match_len + 1 )	// Sync Flush marker
+            if( verbosity >= 0 ) { pp();
-              { rdec.load(); continue; }
+              std::fprintf( stderr, "Unsupported marker code '%d'\n", len ); }
-            if( verbosity >= 0 )
-              {
-              pp();
-              std::fprintf( stderr, "Unsupported marker code '%d'\n", len );
-              }
             return 4;
             }
           }

decoder.h

@@ -106,12 +106,12 @@ public:
     return false;
     }
 
-  bool load( const bool ignore_marking = true )
+  bool load( const bool ignore_nonzero )
     {
     code = 0;
     range = 0xFFFFFFFFU;
-    // check and discard first byte of the LZMA stream
+    // check first byte of the LZMA stream
-    if( get_byte() != 0 && !ignore_marking ) return false;
+    if( get_byte() != 0 && !ignore_nonzero ) return false;
     for( int i = 0; i < 4; ++i ) code = ( code << 8 ) | get_byte();
     return true;
     }
@@ -305,7 +305,7 @@ class LZ_decoder
   unsigned long long stream_position() const
     { return partial_data_pos + stream_pos; }
   void flush_data();
-  int check_trailer( const Pretty_print & pp, const bool ignore_empty ) const;
+  bool check_trailer( const Pretty_print & pp ) const;
 
   uint8_t peek_prev() const
     { return buffer[((pos > 0) ? pos : dictionary_size)-1]; }
@@ -381,7 +381,7 @@ public:
   unsigned crc() const { return crc_ ^ 0xFFFFFFFFU; }
   unsigned long long data_position() const { return partial_data_pos + pos; }
 
-  int decode_member( const Cl_options & cl_opts, const Pretty_print & pp );
+  int decode_member( const Pretty_print & pp, const bool ignore_nonzero );
   int decode_member()
-    { return decode_member( Cl_options(), Pretty_print( "" ) ); }
+    { return decode_member( Pretty_print( "" ), true ); }
   };

doc/lziprecover.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.49.2.
-.TH LZIPRECOVER "1" "January 2024" "lziprecover 1.24" "User Commands"
+.TH LZIPRECOVER "1" "October 2024" "lziprecover 1.25-pre1" "User Commands"
 .SH NAME
 lziprecover \- recovers data from damaged lzip files
 .SH SYNOPSIS
@@ -7,11 +7,8 @@ lziprecover \- recovers data from damaged lzip files
 [\fI\,options\/\fR] [\fI\,files\/\fR]
 .SH DESCRIPTION
 Lziprecover is a data recovery tool and decompressor for files in the lzip
-compressed data format (.lz). Lziprecover is able to repair slightly damaged
+compressed data format (.lz). Lziprecover also provides Forward Error
-files (up to one single\-byte error per member), produce a correct file by
+Correction (FEC) able to repair any kind of file.
-merging the good parts of two or more damaged copies, reproduce a missing
-(zeroed) sector using a reference file, extract data from damaged files,
-decompress files, and test integrity of files.
 .PP
 With the help of lziprecover, losing an entire archive just because of a
 corrupt byte near the beginning is a thing of the past.
@@ -22,9 +19,6 @@ example multimember tar.lz archives.
 Lziprecover provides random access to the data in multimember files; it only
 decompresses the members containing the desired data.
 .PP
-Lziprecover facilitates the management of metadata stored as trailing data
-in lzip files.
-.PP
 Lziprecover is not a replacement for regular backups, but a last line of
 defense for the case where the backups are also damaged.
 .SH OPTIONS
@@ -41,6 +35,12 @@ exit with error status if trailing data
 \fB\-A\fR, \fB\-\-alone\-to\-lz\fR
 convert lzma\-alone files to lzip format
 .TP
+\fB\-b\fR, \fB\-\-block\-size=\fR<bytes>
+make FEC block size a multiple of <bytes>
+.TP
+\fB\-B\fR, \fB\-\-byte\-repair\fR
+try to repair a corrupt byte in file
+.TP
 \fB\-c\fR, \fB\-\-stdout\fR
 write to standard output, keep input files
 .TP
@@ -65,8 +65,17 @@ reference file for \fB\-\-reproduce\fR
 \fB\-f\fR, \fB\-\-force\fR
 overwrite existing output files
 .TP
+\fB\-F\fR, \fB\-\-fec\fR=\fI\,c[N]\/\fR|r|t|l
+create, repair, test, list (using) fec file
+.TP
+\fB\-0\fR .. \fB\-9\fR
+set FEC fragmentation level [default 9]
+.TP
+\fB\-\-fec\-file=\fR<file>[/]
+read fec file from <file> or directory
+.TP
 \fB\-i\fR, \fB\-\-ignore\-errors\fR
-ignore some errors in \fB\-d\fR, \fB\-D\fR, \fB\-l\fR, \fB\-t\fR, \fB\-\-dump\fR
+ignore non\-fatal errors
 .TP
 \fB\-k\fR, \fB\-\-keep\fR
 keep (don't delete) input files
@@ -77,14 +86,20 @@ print (un)compressed file sizes
 \fB\-m\fR, \fB\-\-merge\fR
 repair errors in file using several copies
 .TP
-\fB\-o\fR, \fB\-\-output=\fR<file>
+\fB\-n\fR, \fB\-\-threads=\fR<n>
-place the output into <file>
+set number of threads for fec create [2]
+.TP
+\fB\-o\fR, \fB\-\-output=\fR<file>[/]
+place the output into <file> or directory
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 suppress all messages
 .TP
-\fB\-R\fR, \fB\-\-byte\-repair\fR
+\fB\-r\fR, \fB\-\-recursive\fR
-try to repair a corrupt byte in file
+(fec) operate recursively on directories
+.TP
+\fB\-R\fR, \fB\-\-dereference\-recursive\fR
+(fec) recursively follow symbolic links
 .TP
 \fB\-s\fR, \fB\-\-split\fR
 split multimember file in single\-member files
@@ -104,22 +119,24 @@ remove members, tdata from files in place
 \fB\-\-strip=\fR<list>:d:e:t
 copy files to stdout stripping members given
 .TP
-\fB\-\-empty\-error\fR
+\fB\-\-ignore\-empty\fR
-exit with error status if empty member in file
+ignore empty members in multimember files
 .TP
-\fB\-\-marking\-error\fR
+\fB\-\-ignore\-nonzero\fR
-exit with error status if 1st LZMA byte not 0
+ignore a nonzero first LZMA byte
 .TP
 \fB\-\-loose\-trailing\fR
 allow trailing data seeming corrupt header
 .TP
-\fB\-\-clear\-marking\fR
+\fB\-\-nonzero\-repair\fR
-reset the first LZMA byte of each member
+repair in place a nonzero first LZMA byte
 .PP
 If no file names are given, or if a file is '\-', lziprecover decompresses
 from standard input to standard output.
 Numbers may be followed by a multiplier: k = kB = 10^3 = 1000,
 Ki = KiB = 2^10 = 1024, M = 10^6, Mi = 2^20, G = 10^9, Gi = 2^30, etc...
+The argument to \fB\-\-fec\fR=\fI\,create\/\fR may be a number of blocks (\fB\-Fc20\fR), a
+percentage (\fB\-Fc5\fR%), or a size in bytes (\fB\-Fc10KiB\fR).
 .PP
 To extract all the files from archive 'foo.tar.lz', use the commands
 \&'tar \fB\-xf\fR foo.tar.lz' or 'lziprecover \fB\-cd\fR foo.tar.lz | tar \fB\-xf\fR \-'.

dump_remove.cc

@@ -298,7 +298,7 @@ int remove_members( const std::vector< std::string > & filenames,
 
 /* Set to zero in place the first LZMA byte of each member in each file by
    opening one rw descriptor for each file. */
-int clear_marking( const std::vector< std::string > & filenames,
+int nonzero_repair( const std::vector< std::string > & filenames,
                     const Cl_options & cl_opts )
   {
   long cleared_members = 0;

fec.h

@@ -0,0 +1,297 @@
+/* Lziprecover - Data recovery tool for the lzip format
+   Copyright (C) 2023-2024 Antonio Diaz Diaz.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 2 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+struct le32
+  {
+  enum { size = 4 };
+  uint8_t data[size];
+
+  le32 & operator=( unsigned n )
+    { for( int i = 0; i < size; ++i ) { data[i] = (uint8_t)n; n >>= 8; }
+      return *this; }
+  unsigned val() const
+    { unsigned n = 0;
+      for( int i = size - 1; i >= 0; --i ) { n <<= 8; n += data[i]; }
+      return n; }
+  bool operator==( const le32 & b ) const
+    { return std::memcmp( data, b.data, size ) == 0; }
+  bool operator!=( const le32 & b ) const { return !( *this == b ); }
+  };
+
+
+inline unsigned long long get_le( const uint8_t * const buf, int size )
+  { unsigned long long n = 0;
+    while( --size >= 0 ) { n <<= 8; n += buf[size]; } return n; }
+
+inline unsigned long long ceil_divide( const unsigned long long size,
+                                       const unsigned long block_size )
+  { return size / block_size + ( size % block_size > 0 ); }
+
+inline unsigned long ceil_divide( const unsigned long size,
+                                  const unsigned long block_size )
+  { return size / block_size + ( size % block_size > 0 ); }
+
+inline uint8_t * set_lastbuf( const uint8_t * const prodata,
+                   const unsigned long prodata_size, const unsigned long fbs,
+                   const bool last_is_missing = false )
+  {
+  const unsigned long rest = prodata_size % fbs;
+  if( rest == 0 ) return 0;		// last data block is complete
+  uint8_t * const lastbuf = new uint8_t[fbs];
+  if( last_is_missing ) return lastbuf;	// uninitialized buffer
+  std::memcpy( lastbuf, prodata + ( prodata_size - rest ), rest );
+  std::memset( lastbuf + rest, 0, fbs - rest );
+  return lastbuf;		// copy of last data block padded to fbs bytes
+  }
+
+enum { min_fbs = 512, max_unit_fbs = 1 << 30 };		//   1 GiB
+const unsigned long long max_fbs = 1ULL << 47;		// 128 TiB
+
+inline bool isvalid_fbs( const unsigned long long fbs )
+  { return fbs >= min_fbs && fbs <= max_fbs && fbs % min_fbs == 0; }
+
+struct Coded_fbs				// fec_block_size
+  {
+  enum { size = 2 };
+  uint8_t data[size];			// 11-bit mantissa, 5-bit exponent
+
+  Coded_fbs() {}				// default constructor
+  Coded_fbs( const unsigned long long fbs, const unsigned unit_fbs )
+    {
+    unsigned long long m = fbs;
+    int e = 0;
+    while( m > 2047 || ( m > 1 && e < 9 ) ) { m >>= 1; ++e; }
+    if( m << e < fbs && ++m > 2047 ) { m >>= 1; ++e; }
+    while( ( m << e ) % unit_fbs != 0 ) if( ++m > 2047 ) { m >>= 1; ++e; }
+    if( m == 0 || m > 2047 || e < 9 || e > 40 || m << e < fbs ||
+        !isvalid_fbs( m << e ) || !isvalid_fbs( fbs ) )
+      internal_error( "Coded_fbs: can't fit fec_block_size in packet." );
+    data[0] = m;
+    data[1] = ( e - 9 ) << 3 | m >> 8;
+    }
+
+  void copy( uint8_t * const buf ) const
+    { buf[0] = data[0]; buf[1] = data[1]; }
+
+  unsigned long long val() const
+    {
+    unsigned long long m = ( ( data[1] & 7 ) << 8 ) | data[0];
+    const int e = ( data[1] >> 3 ) + 9;
+    return m << e;
+    }
+  };
+
+enum { fec_magic_l = 4, crc32_l = le32::size };
+const uint8_t fec_magic[4] = { 0xB3, 0xA5, 0xB6, 0xAF };	// ~"LZIP"
+const uint8_t fec_packet_magic[4] = { fec_magic[0], 'F', 'E', 'C' };
+
+inline bool check_fec_magic( const uint8_t * const image_buffer )
+  { return std::memcmp( image_buffer, fec_magic, 4 ) == 0; }
+
+class Packet_base
+  {
+protected:
+  // the packet trailer contains the CRC32 of the payload
+  enum Lengths { trailer_size = crc32_l };
+
+  // header_size must be a multiple of 4 for uint32_t alignment in mul_add
+  const uint8_t * image_;		// header + payload + trailer
+  bool image_is_external;
+
+  Packet_base() : image_is_external( false ) {}
+  explicit Packet_base( const uint8_t * const image_buffer )
+    : image_( image_buffer ), image_is_external( true ) {}
+  ~Packet_base() { if( !image_is_external ) delete[] image_; }
+
+public:
+  const uint8_t * image() const { return image_; }
+  };
+
+
+class Chksum_packet : public Packet_base
+  {
+  enum { current_version = 0 };
+  enum Lengths { version_l = 1, flags_l = 1, prodata_size_l = 8,
+                 prodata_md5_l = 16 };
+  enum Offsets { version_o = fec_magic_l,
+                 flags_o = version_o + version_l,
+                 fbs_o = flags_o + flags_l,
+                 prodata_size_o = fbs_o + Coded_fbs::size,
+                 prodata_md5_o = prodata_size_o + prodata_size_l,
+                 header_crc_o = prodata_md5_o + prodata_md5_l,
+                 header_size = header_crc_o + crc32_l,
+                 crc_array_o = header_size };
+
+  static unsigned compute_header_crc( const uint8_t * const image_buffer )
+    { return crc32.compute_crc( image_buffer, header_crc_o ); }
+
+public:
+  // check image_buffer with check_image before calling this constructor
+  explicit Chksum_packet( const uint8_t * const image_buffer )
+    : Packet_base( image_buffer ) {}
+  Chksum_packet( const uint8_t * const prodata,
+                 const unsigned long prodata_size,
+                 const md5_type & prodata_md5, const Coded_fbs coded_fbs,
+                 const bool gf16_, const bool is_crc_c_ );
+
+  unsigned long long packet_size() const
+    { return ceil_divide( prodata_size(), fec_block_size() ) *
+             sizeof crc_array()[0] + header_size + trailer_size; }
+  unsigned long long prodata_size() const
+    { return get_le( image_ + prodata_size_o, prodata_size_l ); }
+  const md5_type & prodata_md5() const
+    { return *(md5_type *)(image_ + prodata_md5_o); }
+  unsigned long long fec_block_size() const
+    { return ((Coded_fbs *)(image_ + fbs_o))->val(); }
+  static bool check_flags( const uint8_t * const image_buffer )
+    { return image_buffer[flags_o] <= 3; }
+  bool gf16() const { return image_[flags_o] & 2; }
+  bool is_crc_c() const { return image_[flags_o] & 1; }
+  // crc_array contains one CRC32 or one CRC32-C per protected data block
+  const le32 * crc_array() const
+    { return (const le32 *)(image_ + crc_array_o); }
+
+  static unsigned min_packet_size()
+    { return header_size + le32::size + trailer_size; }
+  static uint8_t version( const uint8_t * const image_buffer )
+    { return image_buffer[version_o]; }
+  static bool check_version( const uint8_t * const image_buffer )
+    { return image_buffer[version_o] == current_version; }
+
+  static unsigned check_image( const uint8_t * const image_buffer,
+                               const unsigned long max_size );
+  bool check_payload_crc() const
+    {
+    const unsigned paysize = packet_size() - header_size - trailer_size;
+    const unsigned payload_crc_o = crc_array_o + paysize;
+    const unsigned payload_crc = get_le( image_ + payload_crc_o, crc32_l );
+    return crc32.compute_crc( image_ + crc_array_o, paysize ) == payload_crc;
+    }
+  };
+
+
+class Fec_packet : public Packet_base
+  {
+  enum Lengths { fbn_l = 2 };
+  enum Offsets { fbn_o = fec_magic_l,
+                 fbs_o = fbn_o + fbn_l,
+                 header_crc_o = fbs_o + Coded_fbs::size,
+                 header_size = header_crc_o + crc32_l,
+                 fec_block_o = header_size };
+
+  static unsigned compute_header_crc( const uint8_t * const image_buffer )
+    { return crc32.compute_crc( image_buffer, header_crc_o ); }
+
+public:
+  // check image_buffer with check_image before calling this constructor
+  explicit Fec_packet( const uint8_t * const image_buffer )
+    : Packet_base( image_buffer ) {}
+  Fec_packet( const uint8_t * const prodata, const uint8_t * const lastbuf,
+              const unsigned fbn, const unsigned k,
+              const Coded_fbs coded_fbs, const bool gf16 );
+
+  unsigned long long packet_size() const
+    { return header_size + fec_block_size() + trailer_size; }
+  unsigned fec_block_number() const
+    { return get_le( image_ + fbn_o, fbn_l ); }
+  unsigned long long fec_block_size() const	// number of fec bytes
+    { return ((Coded_fbs *)(image_ + fbs_o))->val(); }
+  const uint8_t * fec_block() const { return image_ + fec_block_o; }
+
+  static unsigned min_packet_size()
+    { return header_size + min_fbs + trailer_size; }
+
+  static unsigned long check_image( const uint8_t * const image_buffer,
+                                    const unsigned long max_size );
+  };
+
+
+enum { max_k8 = 128, max_k16 = 32768, max_nk16 = 2048 };
+const char * const fec_extension = ".fec";
+
+inline void prot_stdin()
+  { show_file_error( "(stdin)", "Can't read protected data from standard input." ); }
+
+// defined in fec_create.cc
+enum { fc_percent, fc_blocks, fc_bytes };
+void cleanup_mutex_lock();
+int gf_check( const unsigned k, const bool cl_gf16, const bool fec_random );
+void extract_dirname( const std::string & name, std::string & srcdir );
+void replace_dirname( const std::string & name, const std::string & srcdir,
+                      const std::string & destdir, std::string & outname );
+bool has_fec_extension( const std::string & name );
+const char * printable_name( const std::string & filename, const bool in = true );
+int fec_create( const std::vector< std::string > & filenames,
+                const std::string & default_output_filename,
+                const unsigned long fb_or_pct, const unsigned cl_block_size,
+                const unsigned num_workers, const char debug_level,
+                const char fctype, const char fec_level, const char recursive,
+                const bool cl_gf16, const bool fec_random, const bool force,
+                const bool to_stdout );
+
+// defined in fec_repair.cc
+int fec_test( const std::vector< std::string > & filenames,
+              const std::string & cl_fec_filename,
+              const std::string & default_output_filename,
+              const char recursive, const bool force, const bool ignore_errors,
+              const bool repair, const bool to_stdout );
+int fec_list( const std::vector< std::string > & filenames,
+              const bool ignore_errors );
+int fec_dc( const std::string & input_filename,
+            const std::string & cl_fec_filename, const unsigned cblocks );
+int fec_dz( const std::string & input_filename,
+            const std::string & cl_fec_filename,
+            std::vector< Block > & range_vector );
+int fec_dZ( const std::string & input_filename,
+            const std::string & cl_fec_filename,
+            const unsigned delta, const int sector_size );
+
+// defined in recursive.cc
+bool next_filename( std::list< std::string > & filelist,
+                    std::string & input_filename, int & retval,
+                    const char recursive );
+
+// defined in gf8.cc, gf16.cc
+void gf8_init();
+void gf16_init();
+bool gf8_check( const std::vector< unsigned > & fbn_vector, const unsigned k );
+bool gf16_check( const std::vector< unsigned > & fbn_vector, const unsigned k );
+
+/* buffer, lastbuf: k blocks of input data, last one possibly padded to fbs.
+   fbn: number of the fec block to be created (fbn < max_k).
+*/
+void rs8_encode( const uint8_t * const buffer, const uint8_t * const lastbuf,
+                 uint8_t * const fec_block, const unsigned long fbs,
+                 const unsigned fbn, const unsigned k );
+void rs16_encode( const uint8_t * const buffer, const uint8_t * const lastbuf,
+                  uint8_t * const fec_block, const unsigned long fbs,
+                  const unsigned fbn, const unsigned k );
+
+/* buffer, lastbuf: k data blocks, those in bb_vector are missing.
+   fecbuf: as many fec blocks as missing data blocks in the order of fbn_vector.
+   The repaired data blocks are written in their place in buffer and lastbuf.
+*/
+void rs8_decode( uint8_t * const buffer, uint8_t * const lastbuf,
+                 const std::vector< unsigned > & bb_vector,
+                 const std::vector< unsigned > & fbn_vector,
+                 uint8_t * const fecbuf, const unsigned long fbs,
+                 const unsigned k );
+void rs16_decode( uint8_t * const buffer, uint8_t * const lastbuf,
+                  const std::vector< unsigned > & bb_vector,
+                  const std::vector< unsigned > & fbn_vector,
+                  uint8_t * const fecbuf, const unsigned long fbs,
+                  const unsigned k );

fec_create.cc

@@ -0,0 +1,615 @@
+/* Lziprecover - Data recovery tool for the lzip format
+   Copyright (C) 2023-2024 Antonio Diaz Diaz.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 2 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#define _FILE_OFFSET_BITS 64
+
+#include <algorithm>
+#include <cerrno>
+#include <climits>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <ctime>
+#include <new>
+#include <list>
+#include <string>
+#include <vector>
+#include <pthread.h>
+#include <stdint.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+
+#include "lzip.h"
+#include "md5.h"
+#include "fec.h"
+
+
+namespace {
+
+void xinit_mutex( pthread_mutex_t * const mutex )
+  {
+  const int errcode = pthread_mutex_init( mutex, 0 );
+  if( errcode )
+    { show_error( "pthread_mutex_init", errcode ); cleanup_and_fail( 1 ); }
+  }
+
+void xinit_cond( pthread_cond_t * const cond )
+  {
+  const int errcode = pthread_cond_init( cond, 0 );
+  if( errcode )
+    { show_error( "pthread_cond_init", errcode ); cleanup_and_fail( 1 ); }
+  }
+
+
+void xdestroy_mutex( pthread_mutex_t * const mutex )
+  {
+  const int errcode = pthread_mutex_destroy( mutex );
+  if( errcode )
+    { show_error( "pthread_mutex_destroy", errcode ); cleanup_and_fail( 1 ); }
+  }
+
+void xdestroy_cond( pthread_cond_t * const cond )
+  {
+  const int errcode = pthread_cond_destroy( cond );
+  if( errcode )
+    { show_error( "pthread_cond_destroy", errcode ); cleanup_and_fail( 1 ); }
+  }
+
+
+void xlock( pthread_mutex_t * const mutex )
+  {
+  const int errcode = pthread_mutex_lock( mutex );
+  if( errcode )
+    { show_error( "pthread_mutex_lock", errcode ); cleanup_and_fail( 1 ); }
+  }
+
+void xunlock( pthread_mutex_t * const mutex )
+  {
+  const int errcode = pthread_mutex_unlock( mutex );
+  if( errcode )
+    { show_error( "pthread_mutex_unlock", errcode ); cleanup_and_fail( 1 ); }
+  }
+
+
+void xwait( pthread_cond_t * const cond, pthread_mutex_t * const mutex )
+  {
+  const int errcode = pthread_cond_wait( cond, mutex );
+  if( errcode )
+    { show_error( "pthread_cond_wait", errcode ); cleanup_and_fail( 1 ); }
+  }
+
+void xsignal( pthread_cond_t * const cond )
+  {
+  const int errcode = pthread_cond_signal( cond );
+  if( errcode )
+    { show_error( "pthread_cond_signal", errcode ); cleanup_and_fail( 1 ); }
+  }
+
+
+unsigned long out_size;
+unsigned deliver_id;		// id of worker writing fec packets to outfd
+unsigned check_counter;
+unsigned wait_counter;
+pthread_mutex_t omutex;
+std::vector< pthread_cond_t > may_deliver;	// worker[i] may write
+pthread_mutex_t cmutex = PTHREAD_MUTEX_INITIALIZER;	// cleanup mutex
+
+
+struct Worker_arg
+  {
+  const uint8_t * prodata;
+  const uint8_t * lastbuf;
+  unsigned fec_blocks;
+  unsigned k;
+  unsigned num_workers;
+  unsigned worker_id;
+  Coded_fbs coded_fbs;
+  bool gf16;
+  };
+
+
+// write a fec packet and pass the token to the next thread
+extern "C" void * worker( void * arg )
+  {
+  const Worker_arg & tmp = *(const Worker_arg *)arg;
+  const uint8_t * const prodata = tmp.prodata;
+  const uint8_t * const lastbuf = tmp.lastbuf;
+  const unsigned fec_blocks = tmp.fec_blocks;
+  const unsigned k = tmp.k;
+  const unsigned num_workers = tmp.num_workers;
+  const unsigned worker_id = tmp.worker_id;
+  const Coded_fbs coded_fbs = tmp.coded_fbs;
+  const bool gf16 = tmp.gf16;
+
+  for( unsigned fbn = worker_id; fbn < fec_blocks; fbn += num_workers )
+    {
+    const Fec_packet fec_packet( prodata, lastbuf, fbn, k, coded_fbs, gf16 );
+    const long packet_size = fec_packet.packet_size();
+    xlock( &omutex );
+    ++check_counter;
+    while( worker_id != deliver_id )
+      { ++wait_counter; xwait( &may_deliver[worker_id], &omutex ); }
+    xlock( &cmutex );				// because of cleanup_and_fail
+    if( writeblock( outfd, fec_packet.image(), packet_size ) != packet_size )
+      { xunlock( &cmutex ); cleanup_and_fail( 1 ); }
+    xunlock( &cmutex );
+    out_size += packet_size;
+    if( ++deliver_id >= num_workers ) deliver_id = 0;
+    xsignal( &may_deliver[deliver_id] );	// allow next worker to write
+    xunlock( &omutex );
+    }
+  return 0;
+  }
+
+
+// start the workers and wait for them to finish.
+bool write_fec_mt( const uint8_t * const prodata,
+                   const uint8_t * const lastbuf,
+                   const unsigned fec_blocks, const unsigned k,
+                   const unsigned num_workers, const Coded_fbs coded_fbs,
+                   const char debug_level, const bool gf16 )
+  {
+  if( debug_level & 2 ) std::fputs( "write_fec_mt.\n", stderr );
+  out_size = 0;
+  deliver_id = 0;
+  check_counter = 0;
+  wait_counter = 0;
+  xinit_mutex( &omutex );
+  may_deliver.resize( num_workers );
+  for( unsigned i = 0; i < may_deliver.size(); ++i )
+    xinit_cond( &may_deliver[i] );
+  std::vector< Worker_arg > worker_args( num_workers );
+  std::vector< pthread_t > worker_threads( num_workers );
+
+  for( unsigned i = 0; i < num_workers; ++i )
+    {
+    worker_args[i].prodata = prodata;
+    worker_args[i].lastbuf = lastbuf;
+    worker_args[i].fec_blocks = fec_blocks;
+    worker_args[i].k = k;
+    worker_args[i].num_workers = num_workers;
+    worker_args[i].worker_id = i;
+    worker_args[i].coded_fbs = coded_fbs;
+    worker_args[i].gf16 = gf16;
+    const int errcode =
+      pthread_create( &worker_threads[i], 0, worker, &worker_args[i] );
+    if( errcode ) { show_error( "Can't create worker threads", errcode );
+                    cleanup_and_fail( 1 ); }
+    }
+
+  for( unsigned i = 0; i < num_workers; ++i )
+    {
+    const int errcode = pthread_join( worker_threads[i], 0 );
+    if( errcode ) { show_error( "Can't join worker threads", errcode );
+                    cleanup_and_fail( 1 ); }
+    }
+
+  for( unsigned i = 0; i < may_deliver.size(); ++i )
+    xdestroy_cond( &may_deliver[i] );
+  xdestroy_mutex( &omutex );
+
+  if( debug_level & 1 )
+    std::fprintf( stderr,
+      "workers started                    %8u\n"
+      "any worker tried to write a packet %8u times\n"
+      "any worker had to wait             %8u times\n",
+      num_workers, check_counter, wait_counter );
+
+  return true;
+  }
+
+
+inline void set_le( uint8_t * const buf, const int size, unsigned long n )
+  { for( int i = 0; i < size; ++i ) { buf[i] = (uint8_t)n; n >>= 8; } }
+
+
+unsigned compute_unit_fbs( const unsigned long prodata_size )
+  {
+  unsigned bs = min_fbs;
+  while( bs < 65536 && 4ULL * bs * bs < prodata_size ) bs <<= 1;
+  return bs;
+  }
+
+unsigned long divide_fbs( const unsigned long size, const unsigned blocks,
+                          const unsigned unit_fbs )
+  {
+  unsigned long long fbs = ceil_divide( size, blocks );	// ULL as max_fbs
+  if( fbs < min_fbs ) fbs = min_fbs;
+  else if( fbs > max_fbs ) fbs = max_fbs;
+  return ceil_divide( fbs, unit_fbs );
+  }
+
+
+Coded_fbs compute_fbs( const unsigned long prodata_size,
+                       const unsigned cl_block_size, const char fec_level )
+  {
+  const unsigned unit_fbs = isvalid_fbs( cl_block_size ) ? cl_block_size :
+                            compute_unit_fbs( prodata_size );
+  const unsigned long max_k = ( fec_level == 0 ) ? max_k8 : max_k16;
+  const unsigned k9 = std::min( ceil_divide( prodata_size, unit_fbs ), max_k );
+  const unsigned long fbsu9 = divide_fbs( prodata_size, k9, unit_fbs );
+  const unsigned long fbsu0 = divide_fbs( prodata_size, max_k8, unit_fbs );
+  const unsigned long a = std::min( (10 - fec_level) * fbsu9, fbsu0 );	// lin
+  const unsigned long b = fbsu0 >> fec_level;	// exp
+  const unsigned long fbsu = std::max( a, b );	// join linear and exponential
+  return Coded_fbs( fbsu * unit_fbs, unit_fbs );
+  }
+
+
+unsigned compute_fec_blocks( const unsigned long prodata_size,
+                             const unsigned long fb_or_pct, const char fctype,
+                             const char fec_level, const Coded_fbs coded_fbs )
+  {
+  const unsigned long fbs = coded_fbs.val();
+  const unsigned prodata_blocks = ceil_divide( prodata_size, fbs );
+  const unsigned long max_k = ( fec_level == 0 ) ? max_k8 : max_k16;
+  if( !isvalid_fbs( fbs ) || prodata_blocks > max_k ) return 0;
+  const unsigned long max_nk = ( fec_level == 0 ) ? max_k8 : max_nk16;
+  unsigned fec_blocks;
+  if( fctype == fc_blocks ) fec_blocks = std::min( max_nk, fb_or_pct );
+  else
+    {
+    unsigned long fec_bytes;
+    if( fctype == fc_percent )
+      { const double pct = std::max( 1UL, std::min( 100000UL, fb_or_pct ) );
+        fec_bytes = (unsigned long)std::ceil( prodata_size * pct / 100000 ); }
+    else if( fctype == fc_bytes )
+      fec_bytes = std::min( fb_or_pct, prodata_size );
+    else return 0;			// unknown fctype, must not happen
+    fec_blocks = std::min( ceil_divide( fec_bytes, fbs ), max_nk );
+    }
+  if( fec_blocks > prodata_blocks ) fec_blocks = prodata_blocks;
+  return fec_blocks;
+  }
+
+
+// return random number between 0 and 32767
+unsigned my_rand( unsigned long & state )
+  {
+  state = state * 1103515245 + 12345;
+  return ( state / 65536 ) % 32768;
+  }
+
+void random_fbn_vector( const unsigned fec_blocks, const bool gf16,
+                        std::vector< unsigned > & fbn_vector )
+  {
+  struct timespec ts;
+  clock_gettime( CLOCK_REALTIME, &ts );
+  unsigned long state = ts.tv_nsec;
+  while( state != 0 && ( state & 1 ) == 0 ) state >>= 1;
+  if( state != 0 ) state *= ts.tv_sec; else state = ts.tv_sec;
+  for( unsigned i = 0; i < fec_blocks; ++i )
+    {
+    again: const unsigned fbn =
+      gf16 ? my_rand( state ) : my_rand( state ) % 128;
+    for( unsigned j = 0; j < fbn_vector.size(); ++j )
+      if( fbn == fbn_vector[j] ) goto again;
+    fbn_vector.push_back( fbn );
+    }
+  }
+
+
+bool write_fec( const char * const input_filename,
+                const uint8_t * const prodata, const unsigned long prodata_size,
+                const unsigned long fb_or_pct, const unsigned cl_block_size,
+                unsigned num_workers, const char debug_level, const char fctype,
+                const char fec_level, const bool cl_gf16, const bool fec_random )
+  {
+  const Coded_fbs coded_fbs =
+    compute_fbs( prodata_size, cl_block_size, fec_level );
+  const unsigned fec_blocks =
+    compute_fec_blocks( prodata_size, fb_or_pct, fctype, fec_level, coded_fbs );
+  if( fec_blocks == 0 ) { show_file_error( input_filename,
+    "Input file is too large for fec protection." ); return false; }
+  if( num_workers > fec_blocks ) num_workers = fec_blocks;
+  const unsigned long fbs = coded_fbs.val();
+  const unsigned prodata_blocks = ceil_divide( prodata_size, fbs );
+  md5_type prodata_md5;
+  compute_md5( prodata, prodata_size, prodata_md5 );
+  unsigned chksum_packet_size;
+  const bool gf16 = cl_gf16 || prodata_blocks > max_k8 || fec_blocks > max_k8;
+  {
+  const Chksum_packet chksum_packet( prodata, prodata_size, prodata_md5,
+    coded_fbs, gf16, false );				// CRC32 array
+  const long packet_size = chksum_packet.packet_size();
+  if( writeblock( outfd, chksum_packet.image(), packet_size ) != packet_size )
+    goto fail;
+  chksum_packet_size = packet_size;
+  }
+  {
+  unsigned long fecdata_size = chksum_packet_size;
+  const uint8_t * const lastbuf = set_lastbuf( prodata, prodata_size, fbs );
+  gf16 ? gf16_init() : gf8_init();		// initialize Galois tables
+  if( fec_random )
+    {
+    std::vector< unsigned > fbn_vector;
+    random_fbn_vector( fec_blocks, gf16, fbn_vector );
+    for( unsigned i = 0; i < fbn_vector.size(); ++i )
+      {
+      const unsigned fbn = fbn_vector[i];
+      const Fec_packet
+        fec_packet( prodata, lastbuf, fbn, prodata_blocks, coded_fbs, gf16 );
+      const long packet_size = fec_packet.packet_size();
+      if( writeblock( outfd, fec_packet.image(), packet_size ) != packet_size )
+        { delete[] lastbuf; goto fail; }
+      fecdata_size += packet_size;
+      }
+    }
+  else if( num_workers > 1 )
+    {
+    if( !write_fec_mt( prodata, lastbuf, fec_blocks, prodata_blocks,
+                       num_workers, coded_fbs, debug_level, gf16 ) )
+      { delete[] lastbuf; goto fail; }
+    fecdata_size += out_size;
+    }
+  else for( unsigned fbn = 0; fbn < fec_blocks; ++fbn )
+    {
+    const Fec_packet
+      fec_packet( prodata, lastbuf, fbn, prodata_blocks, coded_fbs, gf16 );
+    const long packet_size = fec_packet.packet_size();
+    if( writeblock( outfd, fec_packet.image(), packet_size ) != packet_size )
+      { delete[] lastbuf; goto fail; }
+    fecdata_size += packet_size;
+    }
+  delete[] lastbuf;
+  if( ( fecdata_size + chksum_packet_size ) / 2 <= fec_blocks * fbs &&
+      fec_blocks > 1 )			// write the second chksum packet
+    {
+    const Chksum_packet chksum_packet( prodata, prodata_size, prodata_md5,
+      coded_fbs, gf16, true );				// CRC32-C array
+    const long packet_size = chksum_packet.packet_size();
+    if( writeblock( outfd, chksum_packet.image(), packet_size ) != packet_size )
+      goto fail;
+    fecdata_size += packet_size;
+    }
+  if( fecdata_size % 4 != 0 ) internal_error( "fecdata_size % 4 != 0" );
+  if( verbosity >= 1 )
+    std::fprintf( stderr, "  %s: %s bytes, %s fec bytes, %u blocks\n",
+                  printable_name( output_filename, false ),
+                  format_num3( fecdata_size ),
+                  format_num3( fec_blocks * fbs ), fec_blocks );
+  return true;
+  }
+fail:
+  show_file_error( input_filename, "Write error", errno ); return false;
+  }
+
+
+int open_instream2( const std::string & name, struct stat * const in_statsp )
+  {
+  if( !has_fec_extension( name ) )
+    return open_instream( name.c_str(), in_statsp, false, true );
+  if( verbosity >= 0 )
+    std::fprintf( stderr, "%s: %s: Input file already has '%s' suffix, ignored.\n",
+                  program_name, name.c_str(), fec_extension );
+  return -1;
+  }
+
+} // end namespace
+
+
+Chksum_packet::Chksum_packet( const uint8_t * const prodata,
+                 const unsigned long prodata_size,
+                 const md5_type & prodata_md5, const Coded_fbs coded_fbs,
+                 const bool gf16_, const bool is_crc_c_ )
+  {
+  const unsigned long fbs = coded_fbs.val();
+  const unsigned prodata_blocks = ceil_divide( prodata_size, fbs );
+  if( prodata_blocks * fbs < prodata_size )
+    internal_error( "prodata_blocks * fec_block_size < prodata_size" );
+  const unsigned paysize = prodata_blocks * sizeof crc_array()[0];
+  const unsigned packet_size = header_size + paysize + trailer_size;
+  if( paysize <= prodata_blocks || packet_size <= paysize )
+    throw std::bad_alloc();
+  uint8_t * const ip = new uint8_t[packet_size];	// writable image ptr
+  image_ = ip;
+
+  std::memcpy( ip, fec_magic, fec_magic_l );
+  ip[version_o] = current_version;
+  ip[flags_o] = ( gf16_ << 1 ) | is_crc_c_;
+  set_le( ip + prodata_size_o, prodata_size_l, prodata_size );
+  *(md5_type *)(ip + prodata_md5_o) = prodata_md5;
+  coded_fbs.copy( ip + fbs_o );
+  set_le( ip + header_crc_o, crc32_l, compute_header_crc( image_ ) );
+
+  le32 * const crc_arr = (le32 *)(ip + crc_array_o);	// fill crc array
+  unsigned i = 0;
+  if( !is_crc_c_ )					// CRC32
+    for( unsigned long pos = 0; pos < prodata_size; pos += fbs, ++i )
+      crc_arr[i] =
+        crc32.compute_crc( prodata + pos, std::min( fbs, prodata_size - pos ) );
+  else
+    {							// CRC32-C
+    const CRC32 crc32c( true );
+    for( unsigned long pos = 0; pos < prodata_size; pos += fbs, ++i )
+      crc_arr[i] =
+        crc32c.compute_crc( prodata + pos, std::min( fbs, prodata_size - pos ) );
+    }
+  if( i != prodata_blocks )
+    internal_error( "wrong fec_block_size or number of prodata_blocks." );
+
+  // compute CRC32 of payload (crc array)
+  set_le( ip + crc_array_o + paysize, crc32_l,
+          crc32.compute_crc( image_ + crc_array_o, paysize ) );
+  }
+
+
+Fec_packet::Fec_packet( const uint8_t * const prodata,
+                        const uint8_t * const lastbuf,
+                        const unsigned fbn, const unsigned k,
+                        const Coded_fbs coded_fbs, const bool gf16 )
+  {
+  const unsigned long fbs = coded_fbs.val();
+  const unsigned long packet_size = header_size + fbs + trailer_size;
+  if( packet_size <= fbs || !fits_in_size_t( packet_size ) )
+    throw std::bad_alloc();
+  uint8_t * const ip = new uint8_t[packet_size];	// writable image ptr
+  image_ = ip;
+
+  std::memcpy( ip, fec_packet_magic, fec_magic_l );
+  set_le( ip + fbn_o, fbn_l, fbn );
+  coded_fbs.copy( ip + fbs_o );
+  set_le( ip + header_crc_o, crc32_l, compute_header_crc( image_ ) );
+
+  // fill fec array
+  gf16 ? rs16_encode( prodata, lastbuf, ip + fec_block_o, fbs, fbn, k ) :
+         rs8_encode( prodata, lastbuf, ip + fec_block_o, fbs, fbn, k );
+
+  // compute CRC32 of payload (fec array)
+  set_le( ip + fec_block_o + fbs, crc32_l,
+          crc32.compute_crc( image_ + fec_block_o, fbs ) );
+  }
+
+
+void cleanup_mutex_lock()		// make cleanup_and_fail thread-safe
+  { pthread_mutex_lock( &cmutex ); }	// ignore errors to avoid loop
+
+int gf_check( const unsigned k, const bool cl_gf16, const bool fec_random )
+  {
+  std::vector< unsigned > fbn_vector;
+  const bool gf16 = cl_gf16 || k > max_k8;
+  if( fec_random ) random_fbn_vector( k, gf16, fbn_vector );
+  return gf16 ? !gf16_check( fbn_vector, k ) : !gf8_check( fbn_vector, k );
+  }
+
+
+/* if name contains slash(es), copy name into srcdir up to the last slash,
+   removing a leading dot followed by slash(es) */
+void extract_dirname( const std::string & name, std::string & srcdir )
+  {
+  unsigned i = 0;
+  unsigned j = name.size();
+  if( j >= 2 && name[0] == '.' && name[1] == '/' )	// remove leading "./"
+    for( i = 2; i < j && name[i] == '/'; ) ++i;
+  while( j > i && name[j-1] != '/' ) --j;	// remove last component if any
+  if( j > i ) srcdir.assign( name, i, j - i );
+  }
+
+
+// replace prefix srcdir with destdir in name and write result to outname
+void replace_dirname( const std::string & name, const std::string & srcdir,
+                      const std::string & destdir, std::string & outname )
+  {
+  if( srcdir.size() && name.compare( 0, srcdir.size(), srcdir ) != 0 )
+    { if( verbosity >= 0 ) std::fprintf( stderr,
+        "dirname '%s' != '%s'\n", name.c_str(), srcdir.c_str() );
+      internal_error( "srcdir mismatch." ); }
+  outname = destdir;
+  outname.append( name, srcdir.size(), name.size() - srcdir.size() );
+  }
+
+
+bool has_fec_extension( const std::string & name )
+  {
+  const unsigned ext_len = std::strlen( fec_extension );
+  return name.size() > ext_len &&
+         name.compare( name.size() - ext_len, ext_len, fec_extension ) == 0;
+  }
+
+
+const char * printable_name( const std::string & filename, const bool in )
+  {
+  if( filename.empty() || filename == "-" ) return in ? "(stdin)" : "(stdout)";
+  return filename.c_str();
+  }
+
+
+int fec_create( const std::vector< std::string > & filenames,
+                const std::string & default_output_filename,
+                const unsigned long fb_or_pct, const unsigned cl_block_size,
+                const unsigned num_workers, const char debug_level,
+                const char fctype, const char fec_level, const char recursive,
+                const bool cl_gf16, const bool fec_random, const bool force,
+                const bool to_stdout )
+  {
+  const bool to_dir = !to_stdout && default_output_filename.size() &&
+                      default_output_filename.end()[-1] == '/';
+  const bool to_file = !to_stdout && !to_dir && default_output_filename.size();
+  if( ( to_stdout || to_file ) && filenames.size() != 1 )
+    { show_error( "You must specify exactly 1 file when redirecting fec data." );
+      return 1; }
+  if( ( to_stdout || to_file ) && recursive )
+    { show_error( "Can't redirect fec data in recursive mode." ); return 1; }
+  if( to_stdout ) { outfd = STDOUT_FILENO; if( !check_tty_out() ) return 1; }
+  else outfd = -1;
+
+  int retval = 0;
+  const bool one_to_one = !to_stdout && !to_file;
+  for( unsigned i = 0; i < filenames.size(); ++i )
+    {
+    if( filenames[i] == "-" )
+      { prot_stdin(); set_retval( retval, 1 ); continue; }
+    std::string srcdir;			// dirname to be replaced by '-o dir/'
+    if( to_dir ) extract_dirname( filenames[i], srcdir );
+    std::list< std::string > filelist( 1U, filenames[i] );
+    std::string input_filename;
+    while( next_filename( filelist, input_filename, retval, recursive ) )
+      {
+      struct stat in_stats;
+      const int infd = open_instream2( input_filename, &in_stats );
+      if( infd < 0 ) { set_retval( retval, 1 ); continue; }
+
+      const char * const input_filenamep = input_filename.c_str();
+      const long long file_size = lseek( infd, 0, SEEK_END );
+      if( file_size <= 0 )
+        { show_file_error( input_filenamep, "Input file is empty." );
+          set_retval( retval, 2 ); close( infd ); continue; }
+      if( !fits_in_size_t( file_size ) )
+        { show_file_error( input_filenamep, large_file_msg );
+          set_retval( retval, 1 ); close( infd ); continue; }
+      const unsigned long prodata_size = file_size;
+      const uint8_t * const prodata =
+        (const uint8_t *)mmap( 0, prodata_size, PROT_READ, MAP_PRIVATE, infd, 0 );
+      close( infd );
+      if( prodata == MAP_FAILED )
+        { show_file_error( input_filenamep, mmap_msg, errno );
+          set_retval( retval, 1 ); continue; }
+
+      if( one_to_one )
+        {
+        if( to_dir ) replace_dirname( input_filename, srcdir,
+                       default_output_filename, output_filename );
+        else output_filename = input_filename;
+        output_filename += fec_extension; set_signal_handler();
+        if( !open_outstream( force, true, false, true, to_dir ) )
+          { munmap( (void *)prodata, prodata_size );
+            set_retval( retval, 1 ); continue; }
+        if( !check_tty_out() )
+          { set_retval( retval, 1 ); return retval; }	// don't delete a tty
+        }
+      else if( to_file && outfd < 0 )	// open outfd after checking infd
+        {
+        output_filename = default_output_filename; set_signal_handler();
+        if( !open_outstream( force, false ) || !check_tty_out() )
+          return 1;	// check tty only once and don't try to delete a tty
+        }
+
+      // write fec data to output file
+      if( !write_fec( input_filenamep, prodata, prodata_size, fb_or_pct,
+                      cl_block_size, num_workers, debug_level, fctype,
+                      fec_level, cl_gf16, fec_random ) )
+        { munmap( (void *)prodata, prodata_size ); cleanup_and_fail( 1 ); }
+      /* To avoid '-Fc | -Ft' running out of address space, munmap before
+         closing outfd and mmap after reading fec data from stdin */
+      munmap( (void *)prodata, prodata_size );
+      if( !close_outstream( &in_stats ) ) cleanup_and_fail( 1 );
+      }
+    }
+  return retval;
+  }

gf16.cc

@@ -0,0 +1,308 @@
+/* Lziprecover - Data recovery tool for the lzip format
+   Copyright (C) 2023-2024 Antonio Diaz Diaz.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 2 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#define _FILE_OFFSET_BITS 64
+
+#include <cstdio>
+#include <cstring>
+#include <list>
+#include <string>
+#include <vector>
+#include <stdint.h>
+#include <unistd.h>	// STDERR_FILENO
+
+#include "lzip.h"
+#include "md5.h"
+#include "fec.h"
+
+namespace {
+
+const uint16_t u16_one = 1;
+const bool little_endian = *(const uint8_t *)&u16_one == 1;
+inline uint16_t swap_bytes( const uint16_t a )
+  { return ( a >> 8 ) | ( a << 8 ); }
+
+struct Galois16_table		// addition/subtraction is exclusive or
+  {
+  enum { size = 1 << 16, poly = 0x1100B };	// generator polynomial
+  uint16_t * log, * ilog, * mul_tables;
+
+  Galois16_table() : log( 0 ), ilog( 0 ), mul_tables( 0 ) {}
+//  ~Galois16_table() { delete[] mul_tables; delete[] ilog; delete[] log; }
+
+  void init()	// fill log, inverse log, and multiplication tables
+    {
+    if( log ) return;
+    log = new uint16_t[size]; ilog = new uint16_t[size];
+    mul_tables = new uint16_t[3 * 256 * 256];		// LL, LH, HH
+    for( unsigned b = 1, i = 0; i < size - 1; ++i )
+      {
+      log[b] = i;
+      ilog[i] = b;
+      b <<= 1;
+      if( b & size ) b ^= poly;
+      }
+    log[0] = size - 1;	// log(0) is not defined, so use a special value
+    ilog[size-1] = 1;
+
+    uint16_t * p = mul_tables;
+    for( int i = 0; i < 16; i += 8 )
+      for( int j = i; j < 16; j += 8 )
+        for( int a = 0; a < 256 << i; a += 1 << i )
+          for( int b = 0; b < 256 << j; b += 1 << j )
+            *p++ = mul( a, b );
+    }
+
+  uint16_t mul( const uint16_t a, const uint16_t b ) const
+    {
+    if( a == 0 || b == 0 ) return 0;
+    const unsigned sum = log[a] + log[b];
+    return ( sum >= size - 1 ) ? ilog[sum-(size-1)] : ilog[sum];
+//    return ilog[(log[a] + log[b]) % (size-1)];
+    }
+
+  uint16_t inverse( const uint16_t a ) const { return ilog[size-1-log[a]]; }
+  } gf;
+
+
+inline bool check_element( const uint16_t * const A, const uint16_t * const B,
+                    const unsigned k, const unsigned row, const unsigned col )
+  {
+  const uint16_t * pa = A + row * k;
+  const uint16_t * pb = B + col;
+  uint16_t sum = 0;
+  for( unsigned i = 0; i < k; ++i, ++pa, pb += k )
+    sum ^= gf.mul( *pa, *pb );
+  return sum == ( row == col );
+  }
+
+/* Check that A * B = I (A, B, I are square matrices of size k * k).
+   Check just the diagonals for matrices larger than 1024 x 1024. */
+bool check_inverse( const uint16_t * const A, const uint16_t * const B,
+                    const unsigned k )
+  {
+  const bool print = verbosity >= 1 && k > max_k8 && isatty( STDERR_FILENO );
+  for( unsigned row = 0; row < k; ++row )	// multiply A * B
+    {
+    if( k <= 1024 )
+      for( unsigned col = 0; col < k; ++col )
+        { if( !check_element( A, B, k, row, col ) )
+            { if( print && row ) std::fputc( '\n', stderr ); return false; } }
+    else
+      if( !check_element( A, B, k, row, row ) ||
+          !check_element( A, B, k, row, k - 1 - row ) )
+        { if( print && row ) std::fputc( '\n', stderr ); return false; }
+    if( print ) std::fprintf( stderr, "\r%5u rows checked \r", row + 1 );
+    }
+  return true;					// A * B == I
+  }
+
+
+/* Invert in place a matrix of size k * k.
+   This is like Gaussian elimination with a virtual identity matrix:
+   A --some_changes--> I, I --same_changes--> A^-1
+   Galois arithmetic is exact. Swapping rows or columns is not needed. */
+bool invert_matrix( uint16_t * const matrix, const unsigned k )
+  {
+  const bool print = verbosity >= 1 && k > max_k8 && isatty( STDERR_FILENO );
+  for( unsigned row = 0; row < k; ++row )
+    {
+    uint16_t * const pivot_row = matrix + row * k;
+    uint16_t pivot = pivot_row[row];
+    if( pivot == 0 )
+      { if( print && row ) std::fputc( '\n', stderr ); return false; }
+    if( pivot != 1 )				// scale the pivot_row
+      {
+      pivot = gf.inverse( pivot );
+      pivot_row[row] = 1;
+      for( unsigned col = 0; col < k; ++col )
+        pivot_row[col] = gf.mul( pivot_row[col], pivot );
+      }
+    // subtract pivot_row from the other rows
+    for( unsigned row2 = 0; row2 < k; ++row2 )
+      if( row2 != row )
+        {
+        uint16_t * const dst_row = matrix + row2 * k;
+        const uint16_t c = dst_row[row]; dst_row[row] = 0;
+        for( unsigned col = 0; col < k; ++col )
+          dst_row[col] ^= gf.mul( pivot_row[col], c );
+        }
+    if( print ) std::fprintf( stderr, "\r%5u rows inverted\r", row + 1 );
+    }
+  return true;
+  }
+
+
+// create dec_matrix containing only the rows needed and invert it in place
+const uint16_t * init_dec_matrix( const std::vector< unsigned > & bb_vector,
+                                  const std::vector< unsigned > & fbn_vector )
+  {
+  const unsigned bad_blocks = bb_vector.size();
+  uint16_t * const dec_matrix = new uint16_t[bad_blocks * bad_blocks];
+
+  // one row for each missing data block
+  for( unsigned row = 0; row < bad_blocks; ++row )
+    {
+    uint16_t * const dec_row = dec_matrix + row * bad_blocks;
+    const unsigned fbn = fbn_vector[row] | 0x8000;
+    for( unsigned col = 0; col < bad_blocks; ++col )
+      dec_row[col] = gf.inverse( fbn ^ bb_vector[col] );
+    }
+  if( !invert_matrix( dec_matrix, bad_blocks ) )
+    internal_error( "GF(2^16) matrix not invertible." );
+  return dec_matrix;
+  }
+
+#if 0
+/* compute dst[] += c * src[]
+   treat the buffers as arrays of 16-bit Galois values */
+inline void mul_add( const uint8_t * const src, uint8_t * const dst,
+                     const unsigned long fbs, const uint16_t c )
+  {
+  if( c == 0 ) return;				// nothing to add
+  const uint16_t * const src16 = (const uint16_t *)src;
+  uint16_t * const dst16 = (uint16_t *)dst;
+
+  if( little_endian )
+    for( unsigned long i = 0; i < fbs / 2; ++i )
+      dst16[i] ^= gf.mul( src16[i], c );
+  else	// big endian
+    for( unsigned long i = 0; i < fbs / 2; ++i )
+      dst16[i] ^= swap_bytes( gf.mul( swap_bytes( src16[i] ), c ) );
+  }
+#else
+
+/* compute dst[] += c * src[]
+   treat the buffers as arrays of pairs of 16-bit Galois values */
+inline void mul_add( const uint8_t * const src, uint8_t * const dst,
+                     const unsigned long fbs, const uint16_t c )
+  {
+  if( c == 0 ) return;				// nothing to add
+  const int cl = c & 0xFF;	// split factor c into low and high bytes
+  const int ch = c >> 8;
+  // pointers to the four multiplication tables (c.low/high * src.low/high)
+  const uint16_t * LL = &gf.mul_tables[cl * 256];
+  const uint16_t * LH = &gf.mul_tables[65536 + cl * 256];
+  const uint16_t * HL = &gf.mul_tables[65536 + ch];		// step 256
+  const uint16_t * HH = &gf.mul_tables[131072 + ch * 256];
+  uint16_t L[256];		// extract the two tables for factor c
+  uint16_t H[256];
+
+  if( little_endian )
+    for( int i = 0; i < 256; ++i )
+      { L[i] = *LL++ ^ *HL; HL+=256; H[i] = *LH++ ^ *HH++; }
+  else	// big endian
+    for( int i = 0; i < 256; ++i )
+      { H[i] = swap_bytes( *LL++ ^ *HL ); HL+=256;
+        L[i] = swap_bytes( *LH++ ^ *HH++ ); }
+
+  const uint32_t * const src32 = (const uint32_t *)src;
+  uint32_t * const dst32 = (uint32_t *)dst;
+
+  for( unsigned long i = 0; i < fbs / 4; ++i )
+    { const uint32_t s = src32[i];
+      dst32[i] ^= L[s & 0xFF] ^ H[s >> 8 & 0xFF] ^
+                  L[s >> 16 & 0xFF] << 16 ^ H[s >> 24] << 16; }
+  }
+#endif
+
+} // end namespace
+
+
+void gf16_init() { gf.init(); }
+
+bool gf16_check( const std::vector< unsigned > & fbn_vector, const unsigned k )
+  {
+  if( k == 0 ) return true;
+  gf.init();
+  bool good = true;
+  for( unsigned a = 1; a < gf.size; ++a )
+    if( gf.mul( a, gf.inverse( a ) ) != 1 )
+      { good = false;
+        std::fprintf( stderr, "%u * ( 1/%u ) != 1 in GF(2^16)\n", a, a ); }
+  uint16_t * const enc_matrix = new uint16_t[k * k];
+  uint16_t * const dec_matrix = new uint16_t[k * k];
+  const bool random = fbn_vector.size() == k;
+  for( unsigned row = 0; row < k; ++row )
+    {
+    const unsigned fbn = ( random ? fbn_vector[row] : row ) | 0x8000;
+    uint16_t * const enc_row = enc_matrix + row * k;
+    for( unsigned col = 0; col < k; ++col )
+      enc_row[col] = gf.inverse( fbn ^ col );
+    }
+  std::memcpy( dec_matrix, enc_matrix, k * k * sizeof (uint16_t) );
+  if( !invert_matrix( dec_matrix, k ) )
+    { good = false; show_error( "GF(2^16) matrix not invertible." ); }
+  else if( !check_inverse( enc_matrix, dec_matrix, k ) )
+    { good = false; show_error( "GF(2^16) matrix A * A^-1 != I" ); }
+  delete[] dec_matrix;
+  delete[] enc_matrix;
+  return good;
+  }
+
+
+void rs16_encode( const uint8_t * const buffer, const uint8_t * const lastbuf,
+                  uint8_t * const fec_block, const unsigned long fbs,
+                  const unsigned fbn, const unsigned k )
+  {
+  if( !gf.log ) internal_error( "GF(2^16) tables not initialized." );
+  /* The encode matrix is a Hilbert matrix of size k * k with one row per
+     fec block and one column per data block.
+     The value of each element is computed on the fly with inverse. */
+  const unsigned row = fbn | 0x8000;
+  std::memset( fec_block, 0, fbs );
+  for( unsigned col = 0; col < k; ++col )
+    {
+    const uint8_t * const src =
+      ( col < k - (lastbuf != 0) ) ? buffer + col * fbs : lastbuf;
+    mul_add( src, fec_block, fbs, gf.inverse( row ^ col ) );
+    }
+  }
+
+
+void rs16_decode( uint8_t * const buffer, uint8_t * const lastbuf,
+                  const std::vector< unsigned > & bb_vector,
+                  const std::vector< unsigned > & fbn_vector,
+                  uint8_t * const fecbuf, const unsigned long fbs,
+                  const unsigned k )
+  {
+  gf.init();
+  const unsigned bad_blocks = bb_vector.size();
+  for( unsigned col = 0, bi = 0; col < k; ++col )	// reduce
+    {
+    if( bi < bad_blocks && col == bb_vector[bi] ) { ++bi; continue; }
+    const uint8_t * const src =
+      ( col < k - (lastbuf != 0) ) ? buffer + col * fbs : lastbuf;
+    for( unsigned row = 0; row < bad_blocks; ++row )
+      {
+      const unsigned fbn = fbn_vector[row] | 0x8000;
+      mul_add( src, fecbuf + row * fbs, fbs, gf.inverse( fbn ^ col ) );
+      }
+    }
+  const uint16_t * const dec_matrix = init_dec_matrix( bb_vector, fbn_vector );
+  for( unsigned col = 0; col < bad_blocks; ++col )	// solve
+    {
+    const unsigned di = bb_vector[col];
+    uint8_t * const dst =
+      ( di < k - (lastbuf != 0) ) ? buffer + di * fbs : lastbuf;
+    std::memset( dst, 0, fbs );
+    const uint16_t * const dec_row = dec_matrix + col * bad_blocks;
+    for( unsigned row = 0; row < bad_blocks; ++row )
+      mul_add( fecbuf + row * fbs, dst, fbs, dec_row[row] );
+    }
+  delete[] dec_matrix;
+  }

gf8.cc

@@ -0,0 +1,244 @@
+/* Lziprecover - Data recovery tool for the lzip format
+   Copyright (C) 2023-2024 Antonio Diaz Diaz.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 2 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#define _FILE_OFFSET_BITS 64
+
+#include <cstdio>
+#include <cstring>
+#include <list>
+#include <string>
+#include <vector>
+#include <stdint.h>
+
+#include "lzip.h"
+#include "md5.h"
+#include "fec.h"
+
+namespace {
+
+struct Galois8_table		// addition/subtraction is exclusive or
+  {
+  enum { size = 1 << 8, poly = 0x11D };		// generator polynomial
+  uint8_t * log, * ilog, * mul_table;
+
+  Galois8_table() : log( 0 ), ilog( 0 ), mul_table( 0 ) {}
+//  ~Galois8_table() { delete[] mul_table; delete[] ilog; delete[] log; }
+
+  void init()	// fill log, inverse log, and multiplication tables
+    {
+    if( log ) return;
+    log = new uint8_t[size]; ilog = new uint8_t[size];
+    mul_table = new uint8_t[size * size];
+    for( unsigned b = 1, i = 0; i < size - 1; ++i )
+      {
+      log[b] = i;
+      ilog[i] = b;
+      b <<= 1;
+      if( b & size ) b ^= poly;
+      }
+    log[0] = size - 1;	// log(0) is not defined, so use a special value
+    ilog[size-1] = 1;
+
+    for( int i = 1; i < size; ++i )
+      {
+      uint8_t * const mul_row = mul_table + i * size;
+      for( int j = 1; j < size; ++j )
+        mul_row[j] = ilog[(log[i] + log[j]) % (size - 1)];
+      }
+    for( int i = 0; i < size; ++i )
+      mul_table[0 * size + i] = mul_table[i * size + 0] = 0;
+    }
+
+  uint8_t inverse( const uint8_t a ) const { return ilog[size-1-log[a]]; }
+  } gf;
+
+
+// check that A * B = I (A, B, I are square matrices of size k * k)
+bool check_inverse( const uint8_t * const A, const uint8_t * const B,
+                    const unsigned k )
+  {
+  for( unsigned row = 0; row < k; ++row )	// multiply A * B
+    for( unsigned col = 0; col < k; ++col )
+      {
+      const uint8_t * pa = A + row * k;
+      const uint8_t * pb = B + col;
+      uint8_t sum = 0;
+      for( unsigned i = 0; i < k; ++i, ++pa, pb += k )
+        sum ^= gf.mul_table[*pa * gf.size + *pb];
+      if( sum != ( row == col ) ) return false;		// A * B != I
+      }
+  return true;
+  }
+
+
+/* Invert in place a matrix of size k * k.
+   This is like Gaussian elimination with a virtual identity matrix:
+   A --some_changes--> I, I --same_changes--> A^-1
+   Galois arithmetic is exact. Swapping rows or columns is not needed. */
+bool invert_matrix( uint8_t * const matrix, const unsigned k )
+  {
+  for( unsigned row = 0; row < k; ++row )
+    {
+    uint8_t * const pivot_row = matrix + row * k;
+    const uint8_t pivot = pivot_row[row];
+    if( pivot == 0 ) return false;
+    if( pivot != 1 )				// scale the pivot_row
+      {
+      const uint8_t * const mul_row =
+        gf.mul_table + gf.inverse( pivot ) * gf.size;
+      pivot_row[row] = 1;
+      for( unsigned col = 0; col < k; ++col )
+        pivot_row[col] = mul_row[pivot_row[col]];
+      }
+    // subtract pivot_row from the other rows
+    for( unsigned row2 = 0; row2 < k; ++row2 )
+      if( row2 != row )
+        {
+        uint8_t * const dst_row = matrix + row2 * k;
+        const uint8_t c = dst_row[row]; dst_row[row] = 0;
+        const uint8_t * const mul_row = gf.mul_table + c * gf.size;
+        for( unsigned col = 0; col < k; ++col )
+          dst_row[col] ^= mul_row[pivot_row[col]];
+        }
+    }
+  return true;
+  }
+
+
+// create dec_matrix containing only the rows needed and invert it in place
+const uint8_t * init_dec_matrix( const std::vector< unsigned > & bb_vector,
+                                 const std::vector< unsigned > & fbn_vector )
+  {
+  const unsigned bad_blocks = bb_vector.size();
+  uint8_t * const dec_matrix = new uint8_t[bad_blocks * bad_blocks];
+
+  // one row for each missing data block
+  for( unsigned row = 0; row < bad_blocks; ++row )
+    {
+    uint8_t * const dec_row = dec_matrix + row * bad_blocks;
+    const unsigned fbn = fbn_vector[row] | 0x80;
+    for( unsigned col = 0; col < bad_blocks; ++col )
+      dec_row[col] = gf.inverse( fbn ^ bb_vector[col] );
+    }
+  if( !invert_matrix( dec_matrix, bad_blocks ) )
+    internal_error( "GF(2^8) matrix not invertible." );
+  return dec_matrix;
+  }
+
+
+/* compute dst[] += c * src[]
+   treat the buffers as arrays of quadruples of 8-bit Galois values */
+inline void mul_add( const uint8_t * const src, uint8_t * const dst,
+                     const unsigned long fbs, const uint8_t c )
+  {
+  if( c == 0 ) return;				// nothing to add
+  const uint8_t * const mul_row = gf.mul_table + c * gf.size;
+  const uint32_t * const src32 = (const uint32_t *)src;
+  uint32_t * const dst32 = (uint32_t *)dst;
+
+  for( unsigned long i = 0; i < fbs / 4; ++i )
+    { const uint32_t s = src32[i];
+      dst32[i] ^= mul_row[s & 0xFF] ^ mul_row[s >> 8 & 0xFF] << 8 ^
+                  mul_row[s >> 16 & 0xFF] << 16 ^ mul_row[s >> 24] << 24; }
+  }
+
+} // end namespace
+
+
+void gf8_init() { gf.init(); }
+
+bool gf8_check( const std::vector< unsigned > & fbn_vector, const unsigned k )
+  {
+  if( k == 0 ) return true;
+  gf.init();
+  bool good = true;
+  for( unsigned a = 1; a < gf.size; ++a )
+    if( gf.mul_table[a * gf.size + gf.inverse( a )] != 1 )
+      { good = false;
+        std::fprintf( stderr, "%u * ( 1/%u ) != 1 in GF(2^8)\n", a, a ); }
+  uint8_t * const enc_matrix = new uint8_t[k * k];
+  uint8_t * const dec_matrix = new uint8_t[k * k];
+  const bool random = fbn_vector.size() == k;
+  for( unsigned row = 0; row < k; ++row )
+    {
+    const unsigned fbn = ( random ? fbn_vector[row] : row ) | 0x80;
+    uint8_t * const enc_row = enc_matrix + row * k;
+    for( unsigned col = 0; col < k; ++col )
+      enc_row[col] = gf.inverse( fbn ^ col );
+    }
+  std::memcpy( dec_matrix, enc_matrix, k * k );
+  if( !invert_matrix( dec_matrix, k ) )
+    { good = false; show_error( "GF(2^8) matrix not invertible." ); }
+  else if( !check_inverse( enc_matrix, dec_matrix, k ) )
+    { good = false; show_error( "GF(2^8) matrix A * A^-1 != I" ); }
+  delete[] dec_matrix;
+  delete[] enc_matrix;
+  return good;
+  }
+
+
+void rs8_encode( const uint8_t * const buffer, const uint8_t * const lastbuf,
+                 uint8_t * const fec_block, const unsigned long fbs,
+                 const unsigned fbn, const unsigned k )
+  {
+  if( !gf.log ) internal_error( "GF(2^8) tables not initialized." );
+  /* The encode matrix is a Hilbert matrix of size k * k with one row per
+     fec block and one column per data block.
+     The value of each element is computed on the fly with inverse. */
+  const unsigned row = fbn | 0x80;
+  std::memset( fec_block, 0, fbs );
+  for( unsigned col = 0; col < k; ++col )
+    {
+    const uint8_t * const src =
+      ( col < k - (lastbuf != 0) ) ? buffer + col * fbs : lastbuf;
+    mul_add( src, fec_block, fbs, gf.inverse( row ^ col ) );
+    }
+  }
+
+
+void rs8_decode( uint8_t * const buffer, uint8_t * const lastbuf,
+                 const std::vector< unsigned > & bb_vector,
+                 const std::vector< unsigned > & fbn_vector,
+                 uint8_t * const fecbuf, const unsigned long fbs,
+                 const unsigned k )
+  {
+  gf.init();
+  const unsigned bad_blocks = bb_vector.size();
+  for( unsigned col = 0, bi = 0; col < k; ++col )	// reduce
+    {
+    if( bi < bad_blocks && col == bb_vector[bi] ) { ++bi; continue; }
+    const uint8_t * const src =
+      ( col < k - (lastbuf != 0) ) ? buffer + col * fbs : lastbuf;
+    for( unsigned row = 0; row < bad_blocks; ++row )
+      {
+      const unsigned fbn = fbn_vector[row] | 0x80;
+      mul_add( src, fecbuf + row * fbs, fbs, gf.inverse( fbn ^ col ) );
+      }
+    }
+  const uint8_t * const dec_matrix = init_dec_matrix( bb_vector, fbn_vector );
+  for( unsigned col = 0; col < bad_blocks; ++col )	// solve
+    {
+    const unsigned di = bb_vector[col];
+    uint8_t * const dst =
+      ( di < k - (lastbuf != 0) ) ? buffer + di * fbs : lastbuf;
+    std::memset( dst, 0, fbs );
+    const uint8_t * const dec_row = dec_matrix + col * bad_blocks;
+    for( unsigned row = 0; row < bad_blocks; ++row )
+      mul_add( fecbuf + row * fbs, dst, fbs, dec_row[row] );
+    }
+  delete[] dec_matrix;
+  }

lunzcrash.cc

@@ -205,14 +205,15 @@ int lunzcrash_bit( const char * const input_filename,
 
   if( verbosity >= 0 )
     {
-    std::printf( "\n%9ld bytes tested\n%9ld total decompressions"
+    std::printf( "\n%11s bytes tested\n%11s total decompressions"
-                 "\n%9ld decompressions returned with zero status",
+                 "\n%11s decompressions returned with zero status",
-                 positions, decompressions, successes );
+                 format_num3( positions ), format_num3( decompressions ),
+                 format_num3( successes ) );
     if( successes > 0 )
       {
       if( failed_comparisons > 0 )
-        std::printf( ", of which\n%9ld comparisons failed\n",
+        std::printf( ", of which\n%11s comparisons failed\n",
-                     failed_comparisons );
+                     format_num3( failed_comparisons ) );
       else std::fputs( "\n            all comparisons passed\n", stdout );
       }
     else std::fputc( '\n', stdout );
@@ -319,14 +320,15 @@ int lunzcrash_block( const char * const input_filename,
 
   if( verbosity >= 0 )
     {
-    std::printf( "\n%9ld blocks tested\n%9ld total decompressions"
+    std::printf( "\n%11s blocks tested\n%11s total decompressions"
-                 "\n%9ld decompressions returned with zero status",
+                 "\n%11s decompressions returned with zero status",
-                 decompressions, decompressions, successes );
+                 format_num3( decompressions ), format_num3( decompressions ),
+                 format_num3( successes ) );
     if( successes > 0 )
       {
       if( failed_comparisons > 0 )
-        std::printf( ", of which\n%9ld comparisons failed\n",
+        std::printf( ", of which\n%11s comparisons failed\n",
-                     failed_comparisons );
+                     format_num3( failed_comparisons ) );
       else std::fputs( "\n            all comparisons passed\n", stdout );
       }
     else std::fputc( '\n', stdout );
@@ -357,7 +359,7 @@ int md5sum_files( const std::vector< std::string > & filenames )
     while( true )
       {
       const int len = readblock( infd, buffer, buffer_size );
-      if( len != buffer_size && errno ) throw Error( "Read error" );
+      if( len != buffer_size && errno ) throw Error( read_error_msg );
       if( len > 0 ) md5sum.md5_update( buffer, len );
       if( len < buffer_size ) break;
       }

lzip.h

@@ -98,9 +98,6 @@ struct Len_model
   };
 
 
-// defined in main.cc
-extern int verbosity;
-
 class Pretty_print		// requires global var 'int verbosity'
   {
   std::string name_;
@@ -154,13 +151,17 @@ class CRC32
   uint32_t data[256];		// Table of CRCs of all 8-bit messages.
 
 public:
-  CRC32()
+  explicit CRC32( const bool castagnoli = false )
     {
+    const unsigned cpol = 0x82F63B78U;	// CRC32-C  Castagnoli polynomial
+    const unsigned ipol = 0xEDB88320U;	// IEEE 802.3 Ethernet polynomial
+    const unsigned poly = castagnoli ? cpol : ipol;
+
     for( unsigned n = 0; n < 256; ++n )
       {
       unsigned c = n;
       for( int k = 0; k < 8; ++k )
-        { if( c & 1 ) c = 0xEDB88320U ^ ( c >> 1 ); else c >>= 1; }
+        { if( c & 1 ) c = poly ^ ( c >> 1 ); else c >>= 1; }
       data[n] = c;
       }
     }
@@ -179,6 +180,15 @@ public:
       c = data[(c^buffer[i])&0xFF] ^ ( c >> 8 );
     crc = c;
     }
+
+  uint32_t compute_crc( const uint8_t * const buffer,
+                        const unsigned long size ) const
+    {
+    uint32_t crc = 0xFFFFFFFFU;
+    for( unsigned long i = 0; i < size; ++i )
+      crc = data[(crc^buffer[i])&0xFF] ^ ( crc >> 8 );
+    return crc ^ 0xFFFFFFFFU;
+    }
   };
 
 extern const CRC32 crc32;
@@ -313,12 +323,12 @@ struct Cl_options		// command-line options
   {
   bool ignore_empty;
   bool ignore_errors;
-  bool ignore_marking;
+  bool ignore_nonzero;
   bool ignore_trailing;
   bool loose_trailing;
 
   Cl_options()
-    : ignore_empty( true ), ignore_errors( false ), ignore_marking( true ),
+    : ignore_empty( false ), ignore_errors( false ), ignore_nonzero( false ),
       ignore_trailing( true ), loose_trailing( false ) {}
   };
 
@@ -333,6 +343,8 @@ class Block
 
 public:
   Block( const long long p, const long long s ) : pos_( p ), size_( s ) {}
+  Block & assign( const long long p, const long long s )
+    { pos_ = p; size_ = s; return *this; }
 
   long long pos() const { return pos_; }
   long long size() const { return size_; }
@@ -354,6 +366,8 @@ public:
     { return pos_ < b.end() && b.pos_ < end(); }
   bool overlaps( const long long pos, const long long size ) const
     { return pos_ < pos + size && pos < end(); }
+  bool touches( const Block & b ) const		// blocks are mergeable
+    { return pos_ <= b.end() && b.pos_ <= end(); }
 
   Block split( const long long pos );
   };
@@ -410,8 +424,10 @@ const char * const bad_magic_msg = "Bad magic number (file not in lzip format)."
 const char * const bad_dict_msg = "Invalid dictionary size in member header.";
 const char * const corrupt_mm_msg = "Corrupt header in multimember file.";
 const char * const empty_msg = "Empty member not allowed.";
-const char * const marking_msg = "Marking data not allowed.";
+const char * const nonzero_msg = "Nonzero first LZMA byte.";
 const char * const trailing_msg = "Trailing data not allowed.";
+const char * const mmap_msg = "Can't mmap";
+const char * const short_file_msg = "Input file is too short.";
 
 // defined in alone_to_lz.cc
 int alone_to_lz( const int infd, const Pretty_print & pp );
@@ -446,17 +462,13 @@ int dump_members( const std::vector< std::string > & filenames,
                   const bool force, const bool strip, const bool to_stdout );
 int remove_members( const std::vector< std::string > & filenames,
                   const Cl_options & cl_opts, const Member_list & member_list );
-int clear_marking( const std::vector< std::string > & filenames,
+int nonzero_repair( const std::vector< std::string > & filenames,
                     const Cl_options & cl_opts );
 
 // defined in list.cc
 int list_files( const std::vector< std::string > & filenames,
                 const Cl_options & cl_opts );
 
-// defined in lzip_index.cc
-int seek_read( const int fd, uint8_t * const buf, const int size,
-               const long long pos );
-
 // defined in lunzcrash.cc
 int lunzcrash_bit( const char * const input_filename,
                    const Cl_options & cl_opts );
@@ -483,9 +495,11 @@ bool open_outstream( const bool force, const bool protect,
 bool output_file_exists();
 void cleanup_and_fail( const int retval );
 bool check_tty_out();
+void format_trailing_bytes( const uint8_t * const data, const int size,
+                            std::string & msg );
 void set_signal_handler();
 bool close_outstream( const struct stat * const in_statsp );
-std::string insert_fixed( std::string name );
+std::string insert_fixed( std::string name, const bool append_lz = true );
 void show_2file_error( const char * const msg1, const char * const name1,
                        const char * const name2, const char * const msg2 );
 class Range_decoder;

lzip_index.cc

@@ -67,13 +67,10 @@ void Lzip_index::set_num_error( const char * const msg, unsigned long long num )
 
 
 bool Lzip_index::read_header( const int fd, Lzip_header & header,
-                              const long long pos, const bool ignore_marking )
+                              const long long pos )
   {
   if( seek_read( fd, header.data, header.size, pos ) != header.size )
     { set_errno_error( "Error reading member header: " ); return false; }
-  uint8_t byte;
-  if( !ignore_marking && readblock( fd, &byte, 1 ) == 1 && byte != 0 )
-    { error_ = marking_msg; retval_ = 2; return false; }
   return true;
   }
 
@@ -123,8 +120,7 @@ bool Lzip_index::skip_gap( const int fd, unsigned long long & pos,
           { while( i > trailer.size && buffer[i-9] == 0 ) --i; continue; }
         if( member_size > ipos + i || !trailer.check_consistency() ) continue;
         Lzip_header header;
-        if( !read_header( fd, header, ipos + i - member_size,
+        if( !read_header( fd, header, ipos + i - member_size ) ) return false;
-                          cl_opts.ignore_marking ) ) return false;
         if( !header.check( ignore_bad_ds ) ) continue;
         const Lzip_header & header2 = *(const Lzip_header *)( buffer + i );
         const bool full_h2 = bsize - i >= header.size;
@@ -153,8 +149,6 @@ bool Lzip_index::skip_gap( const int fd, unsigned long long & pos,
             { error_ = trailing_msg; retval_ = 2; return false; }
           }
         const unsigned long long data_size = trailer.data_size();
-        if( !cl_opts.ignore_empty && data_size == 0 )
-          { error_ = empty_msg; retval_ = 2; return false; }
         pos = ipos + i - member_size;			// good member
         const unsigned dictionary_size = header.dictionary_size();
         if( dictionary_size_ < dictionary_size )
@@ -192,16 +186,16 @@ Lzip_index::Lzip_index( const int infd, const Cl_options & cl_opts,
   {
   if( insize < 0 )
     { set_errno_error( "Input file is not seekable: " ); return; }
+  Lzip_header header;
+  if( insize >= header.size &&
+      ( !read_header( infd, header, 0 ) ||
+        !check_header( header, ignore_bad_ds ) ) ) return;
   if( insize < min_member_size )
-    { error_ = "Input file is too short."; retval_ = 2; return; }
+    { error_ = short_file_msg; retval_ = 2; return; }
   if( insize > INT64_MAX )
     { error_ = "Input file is too long (2^63 bytes or more).";
       retval_ = 2; return; }
 
-  Lzip_header header;
-  if( !read_header( infd, header, 0, cl_opts.ignore_marking ) ||
-      !check_header( header, ignore_bad_ds ) ) return;
-
   // pos always points to a header or to ( EOF || max_pos )
   unsigned long long pos = ( max_pos > 0 ) ? max_pos : insize;
   while( pos >= min_member_size )
@@ -219,8 +213,7 @@ Lzip_index::Lzip_index( const int infd, const Cl_options & cl_opts,
             continue; else return; }
       set_num_error( "Bad trailer at pos ", pos - trailer.size ); break;
       }
-    if( !read_header( infd, header, pos - member_size, cl_opts.ignore_marking ) )
+    if( !read_header( infd, header, pos - member_size ) ) break;
-      break;
     if( !header.check( ignore_bad_ds ) )		// bad header
       {
       if( ignore_gaps || member_vector.empty() )
@@ -229,8 +222,6 @@ Lzip_index::Lzip_index( const int infd, const Cl_options & cl_opts,
       set_num_error( "Bad header at pos ", pos - member_size ); break;
       }
     const unsigned long long data_size = trailer.data_size();
-    if( !cl_opts.ignore_empty && data_size == 0 )
-      { error_ = empty_msg; retval_ = 2; break; }
     pos -= member_size;					// good member
     const unsigned dictionary_size = header.dictionary_size();
     if( dictionary_size_ < dictionary_size )
@@ -246,6 +237,10 @@ Lzip_index::Lzip_index( const int infd, const Cl_options & cl_opts,
     if( retval_ == 0 ) { error_ = "Can't create file index."; retval_ = 2; }
     return;
     }
+  if( !cl_opts.ignore_empty && member_vector.size() > 1 )
+    for( unsigned long i = 0; i < member_vector.size(); ++i )
+      if( member_vector[i].dblock.size() == 0 )
+        { member_vector.clear(); error_ = empty_msg; retval_ = 2; return; }
   std::reverse( member_vector.begin(), member_vector.end() );
   for( unsigned long i = 0; ; ++i )
     {
@@ -272,7 +267,7 @@ Lzip_index::Lzip_index( const std::vector< int > & infd_vector,
   if( insize < 0 )
     { set_errno_error( "Input file is not seekable: " ); return; }
   if( insize < min_member_size )
-    { error_ = "Input file is too short."; retval_ = 2; return; }
+    { error_ = short_file_msg; retval_ = 2; return; }
   if( insize > INT64_MAX )
     { error_ = "Input file is too long (2^63 bytes or more).";
       retval_ = 2; return; }

lzip_index.h

@@ -28,8 +28,8 @@ class Lzip_index
       : dblock( dpos, dsize ), mblock( mpos, msize ),
         dictionary_size( dict_size ) {}
 
-    bool operator==( const Member & m ) const { return ( mblock == m.mblock ); }
+    bool operator==( const Member & m ) const { return mblock == m.mblock; }
-    bool operator!=( const Member & m ) const { return ( mblock != m.mblock ); }
+    bool operator!=( const Member & m ) const { return mblock != m.mblock; }
     };
 
   // member_vector only contains members with a valid header.
@@ -43,8 +43,7 @@ class Lzip_index
   bool check_header( const Lzip_header & header, const bool ignore_bad_ds );
   void set_errno_error( const char * const msg );
   void set_num_error( const char * const msg, unsigned long long num );
-  bool read_header( const int fd, Lzip_header & header, const long long pos,
+  bool read_header( const int fd, Lzip_header & header, const long long pos );
-                    const bool ignore_marking = true );
   bool read_trailer( const int fd, Lzip_trailer & trailer,
                      const long long pos );
   bool skip_gap( const int fd, unsigned long long & pos,
@@ -94,3 +93,6 @@ public:
   unsigned dictionary_size( const long i ) const
     { return member_vector[i].dictionary_size; }
   };
+
+int seek_read( const int fd, uint8_t * const buf, const int size,
+               const long long pos );

main.cc

@@ -26,12 +26,13 @@
 #include <algorithm>
 #include <cctype>
 #include <cerrno>
-#include <climits>		// SSIZE_MAX
+#include <climits>		// CHAR_BIT, SSIZE_MAX
 #include <csignal>
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
 #include <new>
+#include <list>
 #include <string>
 #include <vector>
 #include <fcntl.h>
@@ -42,8 +43,10 @@
 #if defined __MSVCRT__ || defined __OS2__ || defined __DJGPP__
 #include <io.h>
 #if defined __MSVCRT__
+#include <direct.h>
 #define fchmod(x,y) 0
 #define fchown(x,y,z) 0
+#define mkdir(name,mode) _mkdir(name)
 #define SIGHUP SIGTERM
 #define S_ISSOCK(x) 0
 #ifndef S_IRGRP
@@ -62,6 +65,8 @@
 #include "arg_parser.h"
 #include "lzip.h"
 #include "decoder.h"
+#include "md5.h"
+#include "fec.h"
 
 #ifndef O_BINARY
 #define O_BINARY 0
@@ -77,10 +82,7 @@
 #endif
 
 bool fits_in_size_t( const unsigned long long size )	// fits also in long
-  { return ( sizeof (long) <= sizeof (size_t) && size <= LONG_MAX ) ||
+  { return sizeof (long) <= sizeof (size_t) && size <= LONG_MAX; }
-           ( sizeof (int) <= sizeof (size_t) && size <= INT_MAX ); }
-
-int verbosity = 0;
 
 const char * const program_name = "lziprecover";
 std::string output_filename;	// global vars for output file
@@ -95,33 +97,29 @@ const struct { const char * from; const char * to; } known_extensions[] = {
   { ".tlz", ".tar" },
   { 0,      0      } };
 
-enum Mode { m_none, m_alone_to_lz, m_byte_repair, m_clear_marking,
+enum Mode { m_none, m_alone_to_lz, m_byte_repair, m_check, m_debug_byte_repair,
-            m_debug_byte_repair, m_debug_decompress, m_debug_delay,
+            m_debug_decompress, m_debug_delay, m_decompress, m_dump,
-            m_decompress, m_dump, m_list, m_md5sum, m_merge, m_nrep_stats,
+            m_fec_create, m_fec_repair, m_fec_test, m_fec_list, m_fec_dc,
-            m_range_dec, m_remove, m_reproduce, m_show_packets, m_split,
+            m_fec_dz, m_fec_dZ, m_list, m_md5sum, m_merge, m_nonzero_repair,
-            m_strip, m_test, m_unzcrash_bit, m_unzcrash_block };
+            m_nrep_stats, m_range_dec, m_remove, m_reproduce, m_show_packets,
+            m_split, m_strip, m_test, m_unzcrash_bit, m_unzcrash_block };
 
 /* Variable used in signal handler context.
    It is not declared volatile because the handler never returns. */
 bool delete_output_on_interrupt = false;
 
 
-void show_help()
+void show_help( const long num_online )
   {
   std::printf( "Lziprecover is a data recovery tool and decompressor for files in the lzip\n"
-               "compressed data format (.lz). Lziprecover is able to repair slightly damaged\n"
+               "compressed data format (.lz). Lziprecover also provides Forward Error\n"
-               "files (up to one single-byte error per member), produce a correct file by\n"
+               "Correction (FEC) able to repair any kind of file.\n"
-               "merging the good parts of two or more damaged copies, reproduce a missing\n"
-               "(zeroed) sector using a reference file, extract data from damaged files,\n"
-               "decompress files, and test integrity of files.\n"
                "\nWith the help of lziprecover, losing an entire archive just because of a\n"
                "corrupt byte near the beginning is a thing of the past.\n"
                "\nLziprecover can remove the damaged members from multimember files, for\n"
                "example multimember tar.lz archives.\n"
                "\nLziprecover provides random access to the data in multimember files; it only\n"
                "decompresses the members containing the desired data.\n"
-               "\nLziprecover facilitates the management of metadata stored as trailing data\n"
-               "in lzip files.\n"
                "\nLziprecover is not a replacement for regular backups, but a last line of\n"
                "defense for the case where the backups are also damaged.\n"
                "\nUsage: %s [options] [files]\n", invocation_name );
@@ -130,6 +128,8 @@ void show_help()
                "  -V, --version                 output version information and exit\n"
                "  -a, --trailing-error          exit with error status if trailing data\n"
                "  -A, --alone-to-lz             convert lzma-alone files to lzip format\n"
+               "  -b, --block-size=<bytes>      make FEC block size a multiple of <bytes>\n"
+               "  -B, --byte-repair             try to repair a corrupt byte in file\n"
                "  -c, --stdout                  write to standard output, keep input files\n"
                "  -d, --decompress              decompress, test compressed file integrity\n"
                "  -D, --range-decompress=<n-m>  decompress a range of bytes to stdout\n"
@@ -138,39 +138,54 @@ void show_help()
                "      --lzip-name=<name>        name of lzip executable for --reproduce\n"
                "      --reference-file=<file>   reference file for --reproduce\n"
                "  -f, --force                   overwrite existing output files\n"
-               "  -i, --ignore-errors           ignore some errors in -d, -D, -l, -t, --dump\n"
+               "  -F, --fec=c[N]|r|t|l          create, repair, test, list (using) fec file\n"
+               "  -0 .. -9                      set FEC fragmentation level [default 9]\n"
+               "      --fec-file=<file>[/]      read fec file from <file> or directory\n"
+               "  -i, --ignore-errors           ignore non-fatal errors\n"
                "  -k, --keep                    keep (don't delete) input files\n"
                "  -l, --list                    print (un)compressed file sizes\n"
                "  -m, --merge                   repair errors in file using several copies\n"
-               "  -o, --output=<file>           place the output into <file>\n"
+               "  -n, --threads=<n>             set number of threads for fec create [%ld]\n"
+               "  -o, --output=<file>[/]        place the output into <file> or directory\n"
                "  -q, --quiet                   suppress all messages\n"
-               "  -R, --byte-repair             try to repair a corrupt byte in file\n"
+               "  -r, --recursive               (fec) operate recursively on directories\n"
+               "  -R, --dereference-recursive   (fec) recursively follow symbolic links\n"
                "  -s, --split                   split multimember file in single-member files\n"
                "  -t, --test                    test compressed file integrity\n"
                "  -v, --verbose                 be verbose (a 2nd -v gives more)\n"
                "      --dump=<list>:d:e:t       dump members, damaged/empty, tdata to stdout\n"
                "      --remove=<list>:d:e:t     remove members, tdata from files in place\n"
                "      --strip=<list>:d:e:t      copy files to stdout stripping members given\n"
-               "      --empty-error             exit with error status if empty member in file\n"
+               "      --ignore-empty            ignore empty members in multimember files\n"
-               "      --marking-error           exit with error status if 1st LZMA byte not 0\n"
+               "      --ignore-nonzero          ignore a nonzero first LZMA byte\n"
                "      --loose-trailing          allow trailing data seeming corrupt header\n"
-               "      --clear-marking           reset the first LZMA byte of each member\n" );
+               "      --nonzero-repair          repair in place a nonzero first LZMA byte\n",
+               num_online );
   if( verbosity >= 1 )
     {
     std::printf( "\nDebug options for experts:\n"
                  "  -E, --debug-reproduce=<range>[,ss]  set range to 0 and try to reproduce file\n"
+                 "  -F, --fec=dc<n>                   test repair combinations of n zeroed blocks\n"
+                 "  -F, --fec=dz<range>[:<range>]...  test repair zeroed block(s) at range(s)\n"
+                 "  -F, --fec=dZ<size>[,<delta>]      test repair zeroed blocks of size <size>\n"
                  "  -M, --md5sum                      print the MD5 digests of the input files\n"
                  "  -S, --nrep-stats[=<val>]          print stats of N-byte repeated sequences\n"
                  "  -U, --unzcrash=1|B<size>          test 1-bit or block errors in input file\n"
                  "  -W, --debug-decompress=<pos>,<val>  set pos to val and decompress to stdout\n"
                  "  -X, --show-packets[=<pos>,<val>]  show in stdout the decoded LZMA packets\n"
                  "  -Y, --debug-delay=<range>         find max error detection delay in <range>\n"
-                 "  -Z, --debug-byte-repair=<pos>,<val>  test repair one-byte error at <pos>\n" );
+                 "  -Z, --debug-byte-repair=<pos>,<val>  test repair one-byte error at <pos>\n"
+                 "      --check=<size>                check creation of FEC decode matrix\n"
+                 "      --debug=<level>               print parallel FEC statistics to stderr\n"
+                 "      --gf16                        use GF(2^16) to create fec files\n"
+                 "      --random                      create fec files with random block numbers\n" );
     }
   std::printf( "\nIf no file names are given, or if a file is '-', lziprecover decompresses\n"
                "from standard input to standard output.\n"
                "Numbers may be followed by a multiplier: k = kB = 10^3 = 1000,\n"
                "Ki = KiB = 2^10 = 1024, M = 10^6, Mi = 2^20, G = 10^9, Gi = 2^30, etc...\n"
+               "The argument to --fec=create may be a number of blocks (-Fc20), a\n"
+               "percentage (-Fc5%%), or a size in bytes (-Fc10KiB).\n"
                "\nTo extract all the files from archive 'foo.tar.lz', use the commands\n"
                "'tar -xf foo.tar.lz' or 'lziprecover -cd foo.tar.lz | tar -xf -'.\n"
                "\nExit status: 0 for a normal exit, 1 for environmental problems\n"
@@ -279,13 +294,14 @@ next:
 
 namespace {
 
+const char * const inv_arg_msg = "Invalid argument in";
+
 // Recognized formats: <digit> 'a' m[<match_length>]
 int parse_lzip_level( const char * const arg, const char * const option_name )
   {
   if( *arg == 'a' || std::isdigit( *(const unsigned char *)arg ) ) return *arg;
   if( *arg != 'm' )
-    { show_option_error( arg, "Invalid argument in", option_name );
+    { show_option_error( arg, inv_arg_msg, option_name ); std::exit( 1 ); }
-      std::exit( 1 ); }
   if( arg[1] == 0 ) return -1;
   return -getnum( arg + 1, option_name, 0, min_match_len_limit, max_match_len );
   }
@@ -325,6 +341,55 @@ const char * parse_range( const char * const arg, const char * const pn,
   }
 
 
+// Insert b in its place or merge it with contiguous or overlapping blocks.
+void insert_block_sorted( std::vector< Block > & block_vector, const Block & b )
+  {
+  if( block_vector.empty() || b.pos() > block_vector.back().end() )
+    { block_vector.push_back( b ); return; }	// append at the end
+  const long long pos = b.pos();
+  const long long end = b.end();
+  for( unsigned long i = 0; i < block_vector.size(); ++i )
+    if( end <= block_vector[i].pos() )		// maybe insert b before i
+      {
+      if( end < block_vector[i].pos() &&
+          ( i == 0 || pos > block_vector[i-1].end() ) )
+        { block_vector.insert( block_vector.begin() + i, b ); return; }
+      break;
+      }
+  for( unsigned long i = 0; i < block_vector.size(); ++i )
+    if( block_vector[i].touches( b ) )	// merge b with blocks touching it
+      {
+      unsigned long j = i;	// indexes of first/last mergeable blocks
+      while( j + 1 < block_vector.size() && block_vector[j+1].touches( b ) )
+        ++j;
+      const long long new_pos = std::min( pos, block_vector[i].pos() );
+      const long long new_end = std::max( end, block_vector[j].end() );
+      block_vector[i].assign( new_pos, new_end - new_pos );
+      if( i < j ) block_vector.erase( block_vector.begin() + i + 1,
+                                      block_vector.begin() + j + 1 );
+      break;
+      }
+  }
+
+/* Recognized format: <range>[:<range>]...
+   Allow unordered, overlapping ranges. Return ranges sorted and merged. */
+void parse_range_vector( const char * const arg, const char * const pn,
+                         std::vector< Block > & range_vector )
+  {
+  Block range( 0, 0 );
+  const char * p = arg;
+  while( true )
+    {
+    p = parse_range( p, pn, range );
+    insert_block_sorted( range_vector, range );
+    if( *p == 0 ) return;
+    if( *p == ':' ) { ++p; if( *p == 0 ) return; else continue; }
+    show_option_error( p, "Extra characters in", pn );
+    std::exit( 1 );
+    }
+  }
+
+
 void one_file( const int files )
   {
   if( files != 1 )
@@ -355,6 +420,81 @@ void set_mode( Mode & program_mode, const Mode new_mode )
   }
 
 
+// return true if arg is a non-empty prefix of target
+bool compare_prefix( const char * const arg, const char * const target,
+                     const char * const option_name = 0,
+                     unsigned long * const fb_or_pctp = 0, char * fctypep = 0 )
+  {
+  if( arg[0] == target[0] )
+    for( int i = 1; i < INT_MAX; ++i )
+      {
+      if( arg[i] == 0 ) return true;
+      if( fb_or_pctp && std::isdigit( arg[i] ) )
+        {
+        const char * tail = arg + i;
+        const int llimit = std::strchr( tail, '.' ) ? 0 : 1;
+        *fb_or_pctp = getnum( tail, option_name, 0, llimit, LONG_MAX, &tail );
+        if( *tail == 0 )
+          { if( tail[-1] == 'B' ) { *fctypep = fc_bytes; return true; }
+            if( std::isdigit( tail[-1] ) )
+              { if( *fb_or_pctp <= max_nk16 )
+                  { *fctypep = fc_blocks; return true; }
+                getnum( arg + 1, option_name, 0, 1, max_nk16 ); } }
+        else if( *fb_or_pctp <= 100 && std::isdigit( tail[-1] ) )
+          { if( *tail == '%' && tail[1] == 0 )
+              { *fb_or_pctp *= 1000; *fctypep = fc_percent; return true; }
+            if( *tail == '.' && std::isdigit( *++tail ) )
+              { for( int j = 0; j < 3; ++j ) { *fb_or_pctp *= 10;
+                  if( std::isdigit( *tail ) ) *fb_or_pctp += *tail++ - '0'; }
+                if( *tail >= '5' && *tail <= '9' ) { ++tail; ++*fb_or_pctp; }
+                while( std::isdigit( *tail ) ) { ++tail;
+                  if( *fb_or_pctp == 0 && tail[-1] > '0' ) *fb_or_pctp = 1; }
+                if( *tail == '%' && tail[1] == 0 && *fb_or_pctp <= 100000 &&
+                    *fb_or_pctp > 0 ) { *fctypep = fc_percent; return true; } } }
+        return false;
+        }
+      if( arg[i] != target[i] ) break;
+      }
+  return false;
+  }
+
+
+void parse_fec( const char * const arg, const char * const option_name,
+                Mode & program_mode, unsigned long & fb_or_pct,
+                unsigned & cblocks, unsigned & delta, int & sector_size,
+                std::vector< Block > & range_vector, char & fctype )
+  {
+  if( compare_prefix( arg, "create", option_name, &fb_or_pct, &fctype ) )
+    set_mode( program_mode, m_fec_create );
+  else if( compare_prefix( arg, "repair" ) )
+    set_mode( program_mode, m_fec_repair );
+  else if( compare_prefix( arg, "test" ) )
+    set_mode( program_mode, m_fec_test );
+  else if( compare_prefix( arg, "list" ) )
+    set_mode( program_mode, m_fec_list );
+  else if( arg[0] == 'd' && arg[1] == 'c' )
+    { const char * tail = arg + 2;
+      cblocks = getnum( tail, option_name, 0, 1, max_nk16, &tail );
+      if( *tail != 0 )
+        { show_option_error( arg, inv_arg_msg, option_name ); std::exit( 1 ); }
+      set_mode( program_mode, m_fec_dc ); }
+  else if( arg[0] == 'd' && arg[1] == 'z' )
+    { parse_range_vector( arg + 2, option_name, range_vector );
+      set_mode( program_mode, m_fec_dz ); }
+  else if( arg[0] == 'd' && arg[1] == 'Z' )
+    { const char * tail = arg + 2;
+      sector_size = getnum( tail, option_name, 0, 1, INT_MAX, &tail );
+      if( *tail == 0 ) delta = sector_size;
+      else if( *tail == ',' )
+        delta = getnum( tail + 1, option_name, 0, 1, INT_MAX );
+      else { show_option_error( arg, "Comma expected before delta in",
+                                option_name ); std::exit( 1 ); }
+      set_mode( program_mode, m_fec_dZ ); }
+  else
+    { show_option_error( arg, inv_arg_msg, option_name ); std::exit( 1 ); }
+  }
+
+
 void parse_u( const char * const arg, const char * const option_name,
               Mode & program_mode, int & sector_size )
   {
@@ -363,8 +503,7 @@ void parse_u( const char * const arg, const char * const option_name,
     { set_mode( program_mode, m_unzcrash_block );
       sector_size = getnum( arg + 1, option_name, 0, 1, INT_MAX ); }
   else
-    { show_option_error( arg, "Invalid argument in", option_name );
+    { show_option_error( arg, inv_arg_msg, option_name ); std::exit( 1 ); }
-      std::exit( 1 ); }
   }
 
 
@@ -487,6 +626,9 @@ bool make_dirs( const std::string & name )
 const char * const force_msg =
   "Output file already exists. Use '--force' to overwrite it.";
 
+unsigned char xdigit( const unsigned value )	// hex digit for 'value'
+  { return (value <= 9) ? '0' + value : (value <= 15) ? 'A' + value - 10 : 0; }
+
 } // end namespace
 
 bool open_outstream( const bool force, const bool protect,
@@ -499,8 +641,8 @@ bool open_outstream( const bool force, const bool protect,
   if( force ) flags |= O_TRUNC; else flags |= O_EXCL;
 
   outfd = -1;
-  if( output_filename.size() &&
+  if( output_filename.size() && output_filename.end()[-1] == '/' )
-      output_filename[output_filename.size()-1] == '/' ) errno = EISDIR;
+    errno = EISDIR;
   else {
     if( ( !protect || to_file ) && !make_dirs( output_filename ) )
       { show_file_error( output_filename.c_str(),
@@ -535,6 +677,7 @@ void set_signals( void (*action)(int) )
 
 void cleanup_and_fail( const int retval )
   {
+  cleanup_mutex_lock();		// only one thread can delete and exit
   set_signals( SIG_IGN );			// ignore signals
   if( delete_output_on_interrupt )
     {
@@ -559,6 +702,22 @@ bool check_tty_out()
   return true;
   }
 
+
+void format_trailing_bytes( const uint8_t * const data, const int size,
+                            std::string & msg )
+  {
+  for( int i = 0; i < size; ++i )
+    {
+    msg += xdigit( data[i] >> 4 );
+    msg += xdigit( data[i] & 0x0F );
+    msg += ' ';
+    }
+  msg += '\'';
+  for( int i = 0; i < size; ++i )
+    msg += std::isprint( data[i] ) ? data[i] : '.';
+  msg += '\'';
+  }
+
 namespace {
 
 extern "C" void signal_handler( int )
@@ -617,14 +776,6 @@ void close_and_set_permissions( const struct stat * const in_statsp )
   }
 
 
-unsigned char xdigit( const unsigned value )	// hex digit for 'value'
-  {
-  if( value <= 9 ) return '0' + value;
-  if( value <= 15 ) return 'A' + value - 10;
-  return 0;
-  }
-
-
 bool show_trailing_data( const uint8_t * const data, const int size,
                          const Pretty_print & pp, const bool all,
                          const int ignore_trailing )	// -1 = show
@@ -634,16 +785,7 @@ bool show_trailing_data( const uint8_t * const data, const int size,
     std::string msg;
     if( !all ) msg = "first bytes of ";
     msg += "trailing data = ";
-    for( int i = 0; i < size; ++i )
+    format_trailing_bytes( data, size, msg );
-      {
-      msg += xdigit( data[i] >> 4 );
-      msg += xdigit( data[i] & 0x0F );
-      msg += ' ';
-      }
-    msg += '\'';
-    for( int i = 0; i < size; ++i )
-      { if( std::isprint( data[i] ) ) msg += data[i]; else msg += '.'; }
-    msg += '\'';
     pp( msg.c_str() );
     if( ignore_trailing == 0 ) show_file_error( pp.name(), trailing_msg );
     }
@@ -658,6 +800,7 @@ int decompress( const unsigned long long cfile_size, const int infd,
   unsigned long long partial_file_pos = 0;
   Range_decoder rdec( infd );
   int retval = 0;
+  bool empty = false, nonempty = false;
 
   for( bool first_member = true; ; first_member = false )
     {
@@ -700,7 +843,7 @@ int decompress( const unsigned long long cfile_size, const int infd,
 
     LZ_decoder decoder( rdec, dictionary_size, outfd );
     show_dprogress( cfile_size, partial_file_pos, &rdec, &pp );	// init
-    const int result = decoder.decode_member( cl_opts, pp );
+    const int result = decoder.decode_member( pp, cl_opts.ignore_nonzero );
     partial_file_pos += rdec.member_position();
     if( result != 0 )
       {
@@ -712,16 +855,19 @@ int decompress( const unsigned long long cfile_size, const int infd,
                       "File ends unexpectedly" : "Decoder error",
                       partial_file_pos );
         }
-      else if( result == 5 ) { pp( empty_msg ); break; }
+      else if( result == 5 ) { pp( nonzero_msg ); break; }
-      else if( result == 6 ) { pp( marking_msg ); break; }
       if( cl_opts.ignore_errors ) { pp.reset(); continue; } else break;
       }
+    if( !cl_opts.ignore_empty )
+      { if( decoder.data_position() == 0 ) empty = true; else nonempty = true; }
     if( verbosity >= 2 )
       { std::fputs( testing ? "ok\n" : "done\n", stderr ); pp.reset(); }
     }
   if( verbosity == 1 && retval == 0 )
     std::fputs( testing ? "ok\n" : "done\n", stderr );
   if( retval == 2 && cl_opts.ignore_errors ) retval = 0;
+  if( empty && nonempty && retval == 0 )
+    { show_file_error( pp.name(), empty_msg ); retval = 2; }
   return retval;
   }
 
@@ -739,7 +885,7 @@ bool close_outstream( const struct stat * const in_statsp )
   }
 
 
-std::string insert_fixed( std::string name )
+std::string insert_fixed( std::string name, const bool append_lz )
   {
   if( name.size() > 7 && name.compare( name.size() - 7, 7, ".tar.lz" ) == 0 )
     name.insert( name.size() - 7, "_fixed" );
@@ -747,7 +893,8 @@ std::string insert_fixed( std::string name )
     name.insert( name.size() - 3, "_fixed" );
   else if( name.size() > 4 && name.compare( name.size() - 4, 4, ".tlz" ) == 0 )
     name.insert( name.size() - 4, "_fixed" );
-  else name += "_fixed.lz";
+  else if( append_lz ) name += "_fixed.lz";
+  else name += "_fixed";
   return name;
   }
 
@@ -794,36 +941,63 @@ void show_dprogress( const unsigned long long cfile_size,
 
 int main( const int argc, const char * const argv[] )
   {
+  std::vector< Block > range_vector;
   Block range( 0, 0 );
   int sector_size = INT_MAX;		// default larger than practical range
   Bad_byte bad_byte;
   Member_list member_list;
+  std::string cl_fec_filename;
   std::string default_output_filename;
   const char * lzip_name = "lzip";		// default is lzip
   const char * reference_filename = 0;
+  unsigned long fb_or_pct = 8;	// fec blocks, bytes (B), or 0.001% to 100%
+  unsigned cblocks = 0;		// blocks per combination in fec_dc
+  unsigned cl_block_size = 0;	// make fbs a multiple of this
+  unsigned num_workers = 0;	// start this many worker threads
+  unsigned delta = 0;		// set to 0 to keep gcc 6.1.0 quiet
   Mode program_mode = m_none;
   int lzip_level = 0;		//  0 = test all levels and match lengths
 				// '0'..'9' = level, 'a' = all levels
 				// -5..-273 = match length, -1 = all lengths
   int repeated_byte = -1;	// 0 to 255, or -1 for all values
   Cl_options cl_opts;		// command-line options
+  char debug_level = 0;
+  char fctype = fc_blocks;	// type of value in fb_or_pct
+  char fec_level = 9;		// fec fragmentation level, default = "-9"
+  char recursive = 0;		// 1 = '-r', 2 = '-R'
+  bool cl_gf16 = false;
+  bool fec_random = false;
   bool force = false;
   bool keep_input_files = false;
   bool to_stdout = false;
   if( argc > 0 ) invocation_name = argv[0];
 
-  enum { opt_cm = 256, opt_du, opt_eer, opt_lt, opt_lzl, opt_lzn, opt_mer,
+  enum { opt_chk = 256, opt_dbg, opt_du, opt_ff, opt_g16, opt_ie, opt_inz,
-         opt_ref, opt_rem, opt_st };
+         opt_lt, opt_lzl, opt_lzn, opt_nzr, opt_ref, opt_rem, opt_rnd, opt_st };
   const Arg_parser::Option options[] =
     {
+    { '0', 0,                       Arg_parser::no  },
+    { '1', 0,                       Arg_parser::no  },
+    { '2', 0,                       Arg_parser::no  },
+    { '3', 0,                       Arg_parser::no  },
+    { '4', 0,                       Arg_parser::no  },
+    { '5', 0,                       Arg_parser::no  },
+    { '6', 0,                       Arg_parser::no  },
+    { '7', 0,                       Arg_parser::no  },
+    { '8', 0,                       Arg_parser::no  },
+    { '9', 0,                       Arg_parser::no  },
     { 'a', "trailing-error",        Arg_parser::no  },
     { 'A', "alone-to-lz",           Arg_parser::no  },
+    { 'b', "block-size",            Arg_parser::yes },
+    { 'B', "byte-repair",           Arg_parser::no  },
+    { 'B', "repair",                Arg_parser::no  },
     { 'c', "stdout",                Arg_parser::no  },
     { 'd', "decompress",            Arg_parser::no  },
     { 'D', "range-decompress",      Arg_parser::yes },
     { 'e', "reproduce",             Arg_parser::no  },
     { 'E', "debug-reproduce",       Arg_parser::yes },
     { 'f', "force",                 Arg_parser::no  },
+    { 'F', "fec",                   Arg_parser::yes },
     { 'h', "help",                  Arg_parser::no  },
     { 'i', "ignore-errors",         Arg_parser::no  },
     { 'k', "keep",                  Arg_parser::no  },
@@ -833,8 +1007,8 @@ int main( const int argc, const char * const argv[] )
     { 'n', "threads",               Arg_parser::yes },
     { 'o', "output",                Arg_parser::yes },
     { 'q', "quiet",                 Arg_parser::no  },
-    { 'R', "byte-repair",        Arg_parser::no  },
+    { 'r', "recursive",             Arg_parser::no  },
-    { 'R', "repair",             Arg_parser::no  },
+    { 'R', "dereference-recursive", Arg_parser::no  },
     { 's', "split",                 Arg_parser::no  },
     { 'S', "nrep-stats",            Arg_parser::maybe },
     { 't', "test",                  Arg_parser::no  },
@@ -845,15 +1019,20 @@ int main( const int argc, const char * const argv[] )
     { 'X', "show-packets",          Arg_parser::maybe },
     { 'Y', "debug-delay",           Arg_parser::yes },
     { 'Z', "debug-byte-repair",     Arg_parser::yes },
-    { opt_cm,  "clear-marking",  Arg_parser::no  },
+    { opt_chk, "check",             Arg_parser::yes },
+    { opt_dbg, "debug",             Arg_parser::yes },
     { opt_du,  "dump",              Arg_parser::yes },
-    { opt_eer, "empty-error",    Arg_parser::no  },
+    { opt_ff,  "fec-file",          Arg_parser::yes },
+    { opt_g16, "gf16",              Arg_parser::no  },
+    { opt_ie,  "ignore-empty",      Arg_parser::no  },
+    { opt_inz, "ignore-nonzero",    Arg_parser::no  },
     { opt_lt,  "loose-trailing",    Arg_parser::no  },
     { opt_lzl, "lzip-level",        Arg_parser::yes },
     { opt_lzn, "lzip-name",         Arg_parser::yes },
-    { opt_mer, "marking-error",  Arg_parser::no  },
+    { opt_nzr, "nonzero-repair",    Arg_parser::no  },
     { opt_ref, "reference-file",    Arg_parser::yes },
     { opt_rem, "remove",            Arg_parser::yes },
+    { opt_rnd, "random",            Arg_parser::no  },
     { opt_st,  "strip",             Arg_parser::yes },
     { 0, 0,                         Arg_parser::no  } };
 
@@ -861,6 +1040,11 @@ int main( const int argc, const char * const argv[] )
   if( parser.error().size() )				// bad option
     { show_error( parser.error().c_str(), 0, true ); return 1; }
 
+  const long num_online = std::max( 1L, sysconf( _SC_NPROCESSORS_ONLN ) );
+  long max_workers = sysconf( _SC_THREAD_THREADS_MAX );
+  if( max_workers < 1 || max_workers > INT_MAX / (int)sizeof (pthread_t) )
+    max_workers = INT_MAX / sizeof (pthread_t);
+
   int argind = 0;
   for( ; argind < parser.arguments(); ++argind )
     {
@@ -871,8 +1055,13 @@ int main( const int argc, const char * const argv[] )
     const char * const arg = sarg.c_str();
     switch( code )
       {
+      case '0': case '1': case '2': case '3': case '4': case '5':
+      case '6': case '7': case '8': case '9': fec_level = code - '0'; break;
       case 'a': cl_opts.ignore_trailing = false; break;
       case 'A': set_mode( program_mode, m_alone_to_lz ); break;
+      case 'b': cl_block_size = getnum( arg, pn, 0, min_fbs, max_unit_fbs ) &
+                ( max_unit_fbs - min_fbs ); break;
+      case 'B': set_mode( program_mode, m_byte_repair ); break;
       case 'c': to_stdout = true; break;
       case 'd': set_mode( program_mode, m_decompress ); break;
       case 'D': set_mode( program_mode, m_range_dec );
@@ -881,17 +1070,20 @@ int main( const int argc, const char * const argv[] )
       case 'E': set_mode( program_mode, m_reproduce );
                 parse_range( arg, pn, range, &sector_size ); break;
       case 'f': force = true; break;
-      case 'h': show_help(); return 0;
+      case 'F': parse_fec( arg, pn, program_mode, fb_or_pct, cblocks, delta,
+                           sector_size, range_vector, fctype ); break;
+      case 'h': show_help( num_online ); return 0;
       case 'i': cl_opts.ignore_errors = true; break;
       case 'k': keep_input_files = true; break;
       case 'l': set_mode( program_mode, m_list ); break;
       case 'm': set_mode( program_mode, m_merge ); break;
       case 'M': set_mode( program_mode, m_md5sum ); break;
-      case 'n': break;
+      case 'n': num_workers = getnum( arg, pn, 0, 1, max_workers ); break;
       case 'o': if( sarg == "-" ) to_stdout = true;
                 else { default_output_filename = sarg; } break;
-      case 'q': verbosity = -1; break;
+      case 'q': cl_verbosity = verbosity = -1; break;
-      case 'R': set_mode( program_mode, m_byte_repair ); break;
+      case 'r': recursive = 1; break;
+      case 'R': recursive = 2; break;
       case 's': set_mode( program_mode, m_split ); break;
       case 'S': if( arg[0] ) repeated_byte = getnum( arg, pn, 0, 0, 255 );
                 set_mode( program_mode, m_nrep_stats ); break;
@@ -907,18 +1099,23 @@ int main( const int argc, const char * const argv[] )
                 parse_range( arg, pn, range ); break;
       case 'Z': set_mode( program_mode, m_debug_byte_repair );
                 bad_byte.parse_bb( arg, pn ); break;
-      case opt_cm: set_mode( program_mode, m_clear_marking );
+      case opt_chk: set_mode( program_mode, m_check );
-                   cl_opts.ignore_marking = true; break;
+                    cblocks = getnum( arg, pn, 0, 1, max_k16 ); break;
+      case opt_dbg: debug_level = getnum( arg, pn, 0, 0, 3 ); break;
       case opt_du:  set_mode( program_mode, m_dump );
                     member_list.parse_ml( arg, pn, cl_opts ); break;
-      case opt_eer: cl_opts.ignore_empty = false; break;
+      case opt_ff:  cl_fec_filename = sarg; break;
+      case opt_g16: cl_gf16 = true; break;
+      case opt_ie:  cl_opts.ignore_empty = true; break;
+      case opt_inz: cl_opts.ignore_nonzero = true; break;
       case opt_lt:  cl_opts.loose_trailing = true; break;
       case opt_lzl: lzip_level = parse_lzip_level( arg, pn ); break;
       case opt_lzn: lzip_name = arg; break;
-      case opt_mer: cl_opts.ignore_marking = false; break;
+      case opt_nzr: set_mode( program_mode, m_nonzero_repair ); break;
       case opt_ref: reference_filename = arg; break;
       case opt_rem: set_mode( program_mode, m_remove );
                     member_list.parse_ml( arg, pn, cl_opts ); break;
+      case opt_rnd: fec_random = true; break;
       case opt_st:  set_mode( program_mode, m_strip );
                     member_list.parse_ml( arg, pn, cl_opts ); break;
       default: internal_error( "uncaught option." );
@@ -935,6 +1132,9 @@ int main( const int argc, const char * const argv[] )
     show_error( "You must specify the operation to be performed.", 0, true );
     return 1;
     }
+  if( program_mode != m_decompress && program_mode != m_list &&
+      program_mode != m_test && program_mode != m_range_dec )
+    cl_opts.ignore_empty = true;
 
   std::vector< std::string > filenames;
   bool filenames_given = false;
@@ -954,12 +1154,11 @@ int main( const int argc, const char * const argv[] )
       one_file( filenames.size() );
       return byte_repair( filenames[0], default_output_filename, cl_opts,
                           terminator, force );
-    case m_clear_marking:
+    case m_check: return gf_check( cblocks, cl_gf16, fec_random );
-      at_least_one_file( filenames.size() );
-      return clear_marking( filenames, cl_opts );
     case m_debug_byte_repair:
       one_file( filenames.size() );
-      return debug_byte_repair( filenames[0].c_str(), cl_opts, bad_byte, terminator );
+      return debug_byte_repair( filenames[0].c_str(), cl_opts, bad_byte,
+                                terminator );
     case m_debug_decompress:
       one_file( filenames.size() );
       return debug_decompress( filenames[0].c_str(), cl_opts, bad_byte, false );
@@ -972,6 +1171,30 @@ int main( const int argc, const char * const argv[] )
       at_least_one_file( filenames.size() );
       return dump_members( filenames, default_output_filename, cl_opts,
                       member_list, force, program_mode == m_strip, to_stdout );
+    case m_fec_create:
+      at_least_one_file( filenames.size() );
+      if( num_workers <= 0 ) num_workers = std::min( num_online, max_workers );
+      return fec_create( filenames, default_output_filename, fb_or_pct,
+                    cl_block_size, num_workers, debug_level, fctype, fec_level,
+                    recursive, cl_gf16, fec_random, force, to_stdout );
+    case m_fec_repair:
+    case m_fec_test:
+      at_least_one_file( filenames.size() );
+      return fec_test( filenames, cl_fec_filename, default_output_filename,
+                       recursive, force, cl_opts.ignore_errors,
+                       program_mode == m_fec_repair, to_stdout );
+    case m_fec_list:
+      if( filenames.empty() ) filenames.push_back("-");
+      return fec_list( filenames, cl_opts.ignore_errors );
+    case m_fec_dc:
+      one_file( filenames.size() );
+      return fec_dc( filenames[0], cl_fec_filename, cblocks );
+    case m_fec_dz:
+      one_file( filenames.size() );
+      return fec_dz( filenames[0], cl_fec_filename, range_vector );
+    case m_fec_dZ:
+      one_file( filenames.size() );
+      return fec_dZ( filenames[0], cl_fec_filename, delta, sector_size );
     case m_list: break;
     case m_md5sum: break;
     case m_merge:
@@ -979,6 +1202,9 @@ int main( const int argc, const char * const argv[] )
         { show_error( "You must specify at least 2 files.", 0, true ); return 1; }
       return merge_files( filenames, default_output_filename, cl_opts,
                           terminator, force );
+    case m_nonzero_repair:
+      at_least_one_file( filenames.size() );
+      return nonzero_repair( filenames, cl_opts );
     case m_nrep_stats:
       return print_nrep_stats( filenames, cl_opts, repeated_byte );
     case m_range_dec:

main_common.cc

@@ -15,6 +15,9 @@
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+int cl_verbosity = 0;		// used to silence internal_error if '-q'
+int verbosity = 0;
+
 namespace {
 
 const char * const program_year = "2024";
@@ -29,8 +32,8 @@ void show_version()
   }
 
 
-// separate numbers of 5 or more digits in groups of 3 digits using '_'
+// separate numbers of 6 or more digits in groups of 3 digits using '_'
-const char * format_num3( long long num )
+const char * format_num3p( long long num, const bool raw = false )
   {
   enum { buffers = 8, bufsize = 4 * sizeof num, n = 10 };
   const char * const si_prefix = "kMGTPEZYRQ";
@@ -42,7 +45,7 @@ const char * format_num3( long long num )
   char * p = buf + bufsize - 1;		// fill the buffer backwards
   *p = 0;	// terminator
   const bool negative = num < 0;
-  if( num > 1024 || num < -1024 )
+  if( !raw && ( num > 9999 || num < -9999 ) )
     {
     char prefix = 0;			// try binary first, then si
     for( int i = 0; i < n && num != 0 && num % 1024 == 0; ++i )
@@ -53,7 +56,7 @@ const char * format_num3( long long num )
         { num /= 1000; prefix = si_prefix[i]; }
     if( prefix ) *(--p) = prefix;
     }
-  const bool split = num >= 10000 || num <= -10000;
+  const bool split = num >= 100000 || num <= -100000;
 
   for( int i = 0; ; )
     {
@@ -136,8 +139,8 @@ long long getnum( const char * const arg, const char * const option_name,
     {
     if( verbosity >= 0 )
       std::fprintf( stderr, "%s: '%s': Value out of limits [%s,%s] in "
-                    "option '%s'.\n", program_name, arg, format_num3( llimit ),
+                    "option '%s'.\n", program_name, arg, format_num3p( llimit ),
-                    format_num3( ulimit ), option_name );
+                    format_num3p( ulimit ), option_name );
     std::exit( 1 );
     }
   if( tailp ) *tailp = tail;
@@ -148,7 +151,6 @@ long long getnum( const char * const arg, const char * const option_name,
 
 
 // Recognized formats: <pos>,<value> <pos>,+<value> <pos>,f<value>
-//
 void Bad_byte::parse_bb( const char * const arg, const char * const pn )
   {
   argument = arg;
@@ -166,6 +168,9 @@ void Bad_byte::parse_bb( const char * const arg, const char * const pn )
   }
 
 
+const char * format_num3( long long num ) { return format_num3p( num, true ); }
+
+
 void show_error( const char * const msg, const int errcode, const bool help )
   {
   if( verbosity < 0 ) return;
@@ -191,7 +196,7 @@ void show_file_error( const char * const filename, const char * const msg,
 
 void internal_error( const char * const msg )
   {
-  if( verbosity >= 0 )
+  if( cl_verbosity >= 0 )
     std::fprintf( stderr, "%s: internal error: %s\n", program_name, msg );
   std::exit( 3 );
   }

md5.cc

@@ -178,7 +178,7 @@ void MD5SUM::md5_finish( md5_type & digest )
   md5_update( padding, len );		// pad to 56 mod 64
   md5_update( bits, 8 );		// append data length in bits
 
-  for( int i = 0, j = 0; i < 4; i++, j += 4 )	// store state in digest
+  for( int i = 0, j = 0; i < 4; ++i, j += 4 )	// store state in digest
     {
     digest[j  ] = (uint8_t)state[i];
     digest[j+1] = (uint8_t)(state[i] >>  8);

md5.h

@@ -23,7 +23,7 @@ struct md5_type
   uint8_t data[16];		// 128-bit md5 digest
 
   bool operator==( const md5_type & d ) const
-    { return ( std::memcmp( data, d.data, 16 ) == 0 ); }
+    { return std::memcmp( data, d.data, 16 ) == 0; }
   bool operator!=( const md5_type & d ) const { return !( *this == d ); }
 //  const uint8_t & operator[]( const int i ) const { return data[i]; }
   uint8_t & operator[]( const int i ) { return data[i]; }

merge.cc

@@ -65,7 +65,7 @@ bool file_crc( uint32_t & crc, const int infd, const char * const filename )
     {
     const int rd = readblock( infd, buffer, buffer_size );
     if( rd != buffer_size && errno )
-      { show_file_error( filename, "Error reading input file", errno );
+      { show_file_error( filename, read_error_msg, errno );
         error = true; break; }
     if( rd > 0 )
       crc32.update_buf( crc, buffer, rd );
@@ -153,12 +153,12 @@ bool diff_member( const long long mpos, const long long msize,
         const int size = std::min( (long long)buffer_size, msize - partial_pos );
         const int rd = readblock( fd1, buffer1, size );
         if( rd != size && errno )
-          { show_file_error( filename1, "Error reading input file", errno );
+          { show_file_error( filename1, read_error_msg, errno );
             error = true; break; }
         if( rd > 0 )
           {
           if( readblock( fd2, buffer2, rd ) != rd )
-            { show_file_error( filename2, "Error reading input file", errno );
+            { show_file_error( filename2, read_error_msg, errno );
               error = true; break; }
           for( int i = 0; i < rd; ++i )
             {
@@ -267,8 +267,7 @@ int open_input_files( const std::vector< std::string > & filenames,
         }
       }
     if( tmp < min_member_size )
-      { show_file_error( filenames[i].c_str(), "Input file is too short." );
+      { show_file_error( filenames[i].c_str(), short_file_msg ); return 2; }
-        return 2; }
     if( i == 0 ) insize = tmp;
     else if( insize != tmp )
       { show_2file_error( "Sizes of input files", filenames[0].c_str(),
@@ -524,7 +523,7 @@ bool copy_file( const int infd, const int outfd, const long long max_size )
     if( max_size >= 0 ) rest -= size;
     const int rd = readblock( infd, buffer, size );
     if( rd != size && errno )
-      { show_error( "Error reading input file", errno ); error = true; break; }
+      { show_error( read_error_msg, errno ); error = true; break; }
     if( rd > 0 )
       {
       const int wr = writeblock( outfd, buffer, rd );

mtester.cc

@@ -349,8 +349,6 @@ int LZ_mtester::debug_decode_member( const long long dpos, const long long mpos,
               if( check_trailer( show_packets ? stdout : 0 ) ) return 0;
               return 3;
               }
-            if( len == min_match_len + 1 )	// Sync Flush marker
-              { rdec.load(); continue; }
             return 4;
             }
           }

nrep_stats.cc

@@ -68,13 +68,13 @@ int print_nrep_stats( const std::vector< std::string > & filenames,
       }
     const unsigned long long cdata_size = lzip_index.cdata_size();
     if( !fits_in_size_t( cdata_size ) )		// mmap uses size_t
-      { show_file_error( input_filename, "Input file is too large for mmap." );
+      { show_file_error( input_filename, large_file_msg );
         set_retval( retval, 1 ); close( infd ); continue; }
     const uint8_t * const buffer =
       (const uint8_t *)mmap( 0, cdata_size, PROT_READ, MAP_PRIVATE, infd, 0 );
     close( infd );
     if( buffer == MAP_FAILED )
-      { show_file_error( input_filename, "Can't mmap", errno );
+      { show_file_error( input_filename, mmap_msg, errno );
         set_retval( retval, 1 ); continue; }
     for( long j = 0; j < lzip_index.members(); ++j )
       {

range_dec.cc

@@ -53,7 +53,7 @@ bool decompress_member( const int infd, const Cl_options & cl_opts,
   if( verbosity >= 2 ) pp();
 
   LZ_decoder decoder( rdec, dictionary_size, outfd, outskip, outend );
-  const int result = decoder.decode_member( cl_opts, pp );
+  const int result = decoder.decode_member( pp, cl_opts.ignore_nonzero );
   if( result != 0 )
     {
     if( verbosity >= 0 && result <= 2 )
@@ -141,7 +141,8 @@ int range_decompress( const std::string & input_filename,
   if( range.end() > udata_size )
     range.size( std::max( 0LL, udata_size - range.pos() ) );
   if( range.size() <= 0 )
-    { if( udata_size > 0 ) show_file_error( filename, "Nothing to do." );
+    { if( udata_size > 0 )
+        show_file_error( filename, "Nothing to do; range is empty." );
       return 0; }
 
   if( to_stdout || default_output_filename.empty() ) outfd = STDOUT_FILENO;

recursive.cc

@@ -0,0 +1,122 @@
+/* Lziprecover - Data recovery tool for the lzip format
+   Copyright (C) 2023-2024 Antonio Diaz Diaz.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 2 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#define _FILE_OFFSET_BITS 64
+
+#include <cerrno>
+#include <cstdio>
+#include <cstring>
+#include <list>
+#include <string>
+#include <vector>
+#include <dirent.h>
+#include <stdint.h>
+#include <sys/stat.h>
+
+#include "lzip.h"
+#include "md5.h"
+#include "fec.h"
+
+namespace {
+
+/* Return true if full_name is a regular file without extension .fec
+   or (a link to) a directory. */
+bool test_full_name( const std::string & full_name, const struct stat * stp,
+                     const bool follow )
+  {
+  struct stat st, st2;
+  if( ( follow && stat( full_name.c_str(), &st ) != 0 ) ||
+      ( !follow && lstat( full_name.c_str(), &st ) != 0 ) ) return false;
+  if( S_ISREG( st.st_mode ) ) return !has_fec_extension( full_name );
+  if( !S_ISDIR( st.st_mode ) ) return false;
+
+  std::string prev_dir( full_name );
+  bool loop = ( stp && st.st_ino == stp->st_ino && st.st_dev == stp->st_dev );
+  if( !loop )
+    for( unsigned i = prev_dir.size(); i > 1; )
+      {
+      while( i > 0 && prev_dir[i-1] != '/' ) --i;
+      if( i == 0 ) break;
+      if( i > 1 ) --i;		// remove trailing slash except at root dir
+      prev_dir.resize( i );
+      if( stat( prev_dir.c_str(), &st2 ) != 0 || !S_ISDIR( st2.st_mode ) ||
+          ( st.st_ino == st2.st_ino && st.st_dev == st2.st_dev ) )
+        { loop = true; break; }
+      }
+  if( loop )			// full_name already visited or above tree
+    show_file_error( full_name.c_str(), "warning: recursive directory loop." );
+  return !loop;			// (link to) directory
+  }
+
+} // end namespace
+
+
+/* Return in input_filename the next file name. ('-' is a valid file name).
+   Recursively found files and directories named "fec" are ignored.
+   Set 'retval' to 1 if a directory fails to open. */
+bool next_filename( std::list< std::string > & filelist,
+                    std::string & input_filename, int & retval,
+                    const char recursive )
+  {
+  while( !filelist.empty() )
+    {
+    input_filename = filelist.front();
+    filelist.pop_front();
+    struct stat st;
+    if( stat( input_filename.c_str(), &st ) == 0 && S_ISDIR( st.st_mode ) )
+      {
+      if( recursive )
+        {
+        DIR * const dirp = opendir( input_filename.c_str() );
+        if( !dirp )
+          {
+          show_file_error( input_filename.c_str(), "Can't open directory", errno );
+          if( retval == 0 ) { retval = 1; } continue;
+          }
+        for( unsigned i = input_filename.size();
+             i > 1 && input_filename[i-1] == '/'; --i )
+          input_filename.resize( i - 1 );	// remove trailing slashes
+        struct stat stdot, *stdotp = 0;
+        if( input_filename[0] != '/' )		// relative file name
+          {
+          if( input_filename == "." ) input_filename.clear();
+          if( stat( ".", &stdot ) == 0 && S_ISDIR( stdot.st_mode ) )
+            stdotp = &stdot;
+          }
+        if( input_filename.size() && input_filename != "/" )
+          input_filename += '/';
+        std::list< std::string > tmp_list;
+        while( true )
+          {
+          const struct dirent * const entryp = readdir( dirp );
+          if( !entryp ) { closedir( dirp ); break; }
+          const std::string tmp_name( entryp->d_name );
+          if( tmp_name == "." || tmp_name == ".." || tmp_name == "fec" ||
+              tmp_name == "FEC" ) continue;
+          const std::string full_name( input_filename + tmp_name );
+          if( test_full_name( full_name, stdotp, recursive == 2 ) )
+            tmp_list.push_back( full_name );
+          }
+        filelist.splice( filelist.begin(), tmp_list );
+        }
+      continue;
+      }
+    return true;
+    }
+  input_filename.clear();
+  return false;
+  }

reproduce.cc

@@ -440,7 +440,7 @@ int reproduce_member( uint8_t * const mbuffer, const long msize,
     (const uint8_t *)mmap( 0, rsize, PROT_READ, MAP_PRIVATE, rfd, 0 );
   close( rfd );
   if( rbuf == MAP_FAILED )
-    { show_file_error( reference_filename, "Can't mmap", errno );
+    { show_file_error( reference_filename, mmap_msg, errno );
       return fatal( 1 ); }
 
   const Lzip_header & header = *(const Lzip_header *)mbuffer;
@@ -457,8 +457,8 @@ int reproduce_member( uint8_t * const mbuffer, const long msize,
 
   const long offset = match_file( *master, rbuf, rsize, reference_filename );
   if( offset < 0 ) { delete master; return 2; }		// no match
-  // Reference data from offset must be at least as large as zeroed sector
+  /* Reference data from offset must be at least as large as zeroed sector
-  // minus member trailer if trailer is inside the zeroed sector.
+     minus member trailer if trailer is inside the zeroed sector. */
   const int t = ( begin + size >= msize ) ? 16 + Lzip_trailer::size : 0;
   if( rsize - offset < size - t )
     { show_file_error( reference_filename, "Not enough reference data after match." );
@@ -567,7 +567,7 @@ int reproduce_file( const std::string & input_filename,
     uint8_t * const mbuffer_base = (uint8_t *)mmap( 0, msize + mpos_rem,
               PROT_READ | PROT_WRITE, MAP_PRIVATE, infd, mpos - mpos_rem );
     if( mbuffer_base == MAP_FAILED )
-      { show_file_error( filename, "Can't mmap", errno ); return 1; }
+      { show_file_error( filename, mmap_msg, errno ); return 1; }
     uint8_t * const mbuffer = mbuffer_base + mpos_rem;
     long size = 0;
     uint8_t value = 0;
@@ -627,7 +627,8 @@ int reproduce_file( const std::string & input_filename,
       std::fputs( "One member reproduced."
                   " Copy of input file still contains errors.\n", stdout );
     else
-      std::fputs( "Copy of input file reproduced successfully.\n", stdout );
+      std::printf( "Repaired copy of '%s' written to '%s'\n",
+                   filename, output_filename.c_str() );
     }
   return 0;
   }
@@ -686,7 +687,7 @@ int debug_reproduce_file( const char * const input_filename,
       uint8_t * const mbuffer_base = (uint8_t *)mmap( 0, msize + mpos_rem,
                 PROT_READ | PROT_WRITE, MAP_PRIVATE, infd, mpos - mpos_rem );
       if( mbuffer_base == MAP_FAILED )
-        { show_file_error( input_filename, "Can't mmap", errno ); return 1; }
+        { show_file_error( input_filename, mmap_msg, errno ); return 1; }
       uint8_t * const mbuffer = mbuffer_base + mpos_rem;
       if( !md5_valid )
         {
@@ -762,18 +763,18 @@ int debug_reproduce_file( const char * const input_filename,
 done:
   if( verbosity >= 0 )
     {
-    std::printf( "\n%9ld sectors tested"
+    std::printf( "\n%11s sectors tested"
-                 "\n%9ld reproductions returned with zero status",
+                 "\n%11s reproductions returned with zero status",
-                 positions, successes );
+                 format_num3( positions ), format_num3( successes ) );
     if( successes > 0 )
       {
       if( failed_comparisons > 0 )
-        std::printf( ", of which\n%9ld comparisons failed\n",
+        std::printf( ", of which\n%11s comparisons failed\n",
-                     failed_comparisons );
+                     format_num3( failed_comparisons ) );
       else std::fputs( "\n            all comparisons passed\n", stdout );
       if( alternative_reproductions > 0 )
-        std::printf( "%9ld alternative reproductions found\n",
+        std::printf( "%11s alternative reproductions found\n",
-                     alternative_reproductions );
+                     format_num3( alternative_reproductions ) );
       }
     else std::fputc( '\n', stdout );
     if( fatal_retval )

testsuite/test.txt

@@ -1,8 +1,7 @@
                     GNU GENERAL PUBLIC LICENSE
                        Version 2, June 1991
 
- Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc. <http://fsf.org/>
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 
@@ -339,8 +338,7 @@ Public License instead of this License.
                     GNU GENERAL PUBLIC LICENSE
                        Version 2, June 1991
 
- Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc. <http://fsf.org/>
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 

testsuite/test_3m.txt.lz.md5

@@ -1 +1 @@
-6a6bb58464ec8567eab17015064d0c5b  test_3m.txt.lz
+aa8ca65001d627f89e7494fa829e710f  test_3m.txt.lz

testsuite/test_bad6.txt

@@ -1,6 +1,3 @@
-)  You can apply it to
-your programs, too.
-
   When we speak of free software, we are referring to freedom, not
 price.  Our General Public Licenses are designed to make sure that you
 have the freedom to distribute copies of free software (and charge for
@@ -23,4 +20,10 @@ rights.
 (2) offer you this license which gives you legal permission to copy,
 distribute and/or modify the software.
 
-  Also, for each author's protection and ours, we want to
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+

testsuite/test_bad7.txt

@@ -1,13 +1,3 @@
-, so
-that any problems introduced by others will not reflect on the original
-authors' reputations.
-
-  Finally, any free program is threatened constantly by software
-patents.  We wish to avoid the danger that redistributors of a free
-program will individually obtain patent licenses, in effect making the
-program proprietary.  To prevent this, we have made it clear that any
-patent must be licensed for everyone's free use or not licensed at all.
-
   The precise terms and conditions for copying, distribution and
 modification follow.
 
@@ -213,3 +203,21 @@ of promoting the sharing and reuse of software generally.
                             NO WARRANTY
 
   11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.

testsuite/test_bad9.txt

@@ -1,5 +1,13 @@
-General
-Public License instead of this License.
                     GNU GENERAL PUBLIC LICENSE
                        Version 2, June 1991
 
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This

unzcrash.cc

@@ -26,7 +26,7 @@
 
 #include <algorithm>
 #include <cerrno>
-#include <climits>		// SSIZE_MAX
+#include <climits>		// CHAR_BIT, SSIZE_MAX
 #include <csignal>
 #include <cstdio>
 #include <cstdlib>
@@ -54,8 +54,6 @@ namespace {
 const char * const program_name = "unzcrash";
 const char * invocation_name = program_name;		// default value
 
-int verbosity = 0;
-
 
 void show_help()
   {
@@ -142,28 +140,29 @@ uint8_t * read_file( const char * const filename, long * const file_sizep )
 
   long buffer_size = 65536;
   uint8_t * buffer = (uint8_t *)std::malloc( buffer_size );
-  if( !buffer ) { show_error( mem_msg ); return 0; }
+  if( !buffer ) { show_file_error( filename, mem_msg ); return 0; }
   long file_size = std::fread( buffer, 1, buffer_size, f );
   while( file_size >= buffer_size || ( !std::ferror( f ) && !std::feof( f ) ) )
     {
     if( file_size >= buffer_size )	// may be false because of EINTR
       {
       if( buffer_size >= LONG_MAX )
-        { show_file_error( filename, "Input file is larger than LONG_MAX." );
+        { show_file_error( filename, large_file_msg );
           std::free( buffer ); return 0; }
       buffer_size = ( buffer_size <= LONG_MAX / 2 ) ? 2 * buffer_size : LONG_MAX;
       uint8_t * const tmp = (uint8_t *)std::realloc( buffer, buffer_size );
-      if( !tmp ) { show_error( mem_msg ); std::free( buffer ); return 0; }
+      if( !tmp )
+        { show_file_error( filename, mem_msg ); std::free( buffer ); return 0; }
       buffer = tmp;
       }
     file_size += std::fread( buffer + file_size, 1, buffer_size - file_size, f );
     }
   if( std::ferror( f ) || !std::feof( f ) )
-    {
+    { show_file_error( filename, read_error_msg, errno );
-    show_file_error( filename, "Error reading input file", errno );
+      std::free( buffer ); return 0; }
-    std::free( buffer ); return 0;
+  if( std::fclose( f ) != 0 )
-    }
+    { show_file_error( filename, "Error closing input file", errno );
-  std::fclose( f );
+      std::free( buffer ); return 0; }
   *file_sizep = file_size;
   return buffer;
   }
@@ -173,13 +172,13 @@ class Bitset8			// 8 value bitset (1 to 8)
   {
   bool data[8];
   static bool valid_digit( const unsigned char ch )
-    { return ( ch >= '1' && ch <= '8' ); }
+    { return ch >= '1' && ch <= '8'; }
 
 public:
   Bitset8() { for( int i = 0; i < 8; ++i ) data[i] = true; }
 
   bool includes( const int i ) const
-    { return ( i >= 1 && i <= 8 && data[i-1] ); }
+    { return i >= 1 && i <= 8 && data[i-1]; }
 
   // Recognized formats: 1 1,2,3 1-4 1,3-5,8 1-3,5-8
   void parse_bs( const char * const arg, const char * const option_name )
@@ -383,7 +382,7 @@ int main( const int argc, const char * const argv[] )
     { 'v', "verbose",   Arg_parser::no  },
     { 'V', "version",   Arg_parser::no  },
     { 'z', "zcmp",      Arg_parser::yes },
-    {  0 , 0,           Arg_parser::no  } };
+    { 0, 0,             Arg_parser::no  } };
 
   const Arg_parser parser( argc, argv, options );
   if( parser.error().size() )				// bad option
@@ -398,15 +397,15 @@ int main( const int argc, const char * const argv[] )
     const char * const arg = parser.argument( argind ).c_str();
     switch( code )
       {
-      case 'h': show_help(); return 0;
       case 'b': bits.parse_bs( arg, pn ); program_mode = m_byte; break;
       case 'B': if( arg[0] ) parse_block( arg, pn, block_size, block_value );
                 program_mode = m_block; break;
       case 'd': delta = getnum( arg, pn, block_size, 1, INT_MAX ); break;
       case 'e': bad_byte.parse_bb( arg, pn ); break;
+      case 'h': show_help(); return 0;
       case 'n': check = false; break;
       case 'p': pos = getnum( arg, pn, block_size, -LONG_MAX, LONG_MAX ); break;
-      case 'q': verbosity = -1; break;
+      case 'q': cl_verbosity = verbosity = -1; break;
       case 's': max_size = getnum( arg, pn, block_size, -LONG_MAX, LONG_MAX ); break;
       case 't': program_mode = m_truncate; break;
       case 'v': if( verbosity < 4 ) ++verbosity; break;
@@ -419,7 +418,8 @@ int main( const int argc, const char * const argv[] )
   if( parser.arguments() - argind != 2 )
     {
     if( verbosity >= 0 )
-      std::fprintf( stderr, "Usage: %s 'lzip -t' file.lz\n", invocation_name );
+      std::fprintf( stderr, "Usage: %s [options] 'lzip -t' file.lz\n",
+                    invocation_name );
     return 1;
     }
 
@@ -532,7 +532,7 @@ int main( const int argc, const char * const argv[] )
       }
   else if( program_mode == m_block )
     {
-    uint8_t * block = (uint8_t *)std::malloc( block_size );
+    uint8_t * const block = (uint8_t *)std::malloc( block_size );
     if( !block ) { show_error( mem_msg ); return 1; }
     for( long i = pos; i < end; i += std::min( delta, end - i ) )
       {
@@ -611,16 +611,17 @@ int main( const int argc, const char * const argv[] )
 
   if( verbosity >= 0 )
     {
-    std::fprintf( stderr, "\n%9ld %ss tested\n%9ld total decompressions"
+    std::fprintf( stderr, "\n%11s %ss tested\n%11s total decompressions"
-                          "\n%9ld decompressions returned with zero status",
+                          "\n%11s decompressions returned with zero status",
-                  positions, mode_str[program_mode], decompressions, successes );
+                  format_num3( positions ), mode_str[program_mode],
+                  format_num3( decompressions ), format_num3( successes ) );
     if( successes > 0 )
       {
       if( zcmp_command.empty() )
         std::fputs( "\n            comparisons disabled\n", stderr );
       else if( failed_comparisons > 0 )
-        std::fprintf( stderr, ", of which\n%9ld comparisons failed\n",
+        std::fprintf( stderr, ", of which\n%11s comparisons failed\n",
-                      failed_comparisons );
+                      format_num3( failed_comparisons ) );
       else std::fputs( "\n            all comparisons passed\n", stderr );
       }
     else std::fputc( '\n', stderr );