Merging upstream version 0.21.

Signed-off-by: Daniel Baumann <daniel@debian.org>

ChangeLog

@@ -1,3 +1,15 @@
+2021-06-14  Antonio Diaz Diaz  <antonio@gnu.org>
+
+	* Version 0.21 released.
+	* Lzlib 1.12 or newer is now required.
+	* decode.cc (decode): Skip members without name except when listing.
+	  decode_lz.cc (dworker): Likewise. (Reported by Florian Schmaus).
+	* New options '-z, --compress' and '-o, --output'.
+	* New option '--warn-newer'.
+	* tarlz.texi (Portable character set): Link to moe section on Unicode.
+	  (Invoking tarlz): Document concatenation to standard output.
+	* check.sh: Fix the '--diff' test on OS/2.
+
 2021-01-08  Antonio Diaz Diaz  <antonio@gnu.org>
 
 	* Version 0.19 released.

INSTALL

@@ -5,8 +5,7 @@ You will need a C++11 compiler and the compression library lzlib installed.
 I use gcc 6.1.0 and 4.1.2, but the code should compile with any standards
 compliant compiler.
 
-Lzlib must be version 1.8 or newer, but --keep-damaged requires lzlib 1.11
+Lzlib must be version 1.12 or newer.
-or newer to recover as much data as possible from each damaged member.
 
 Gcc is available at http://gcc.gnu.org.
 Lzlib is available at http://www.nongnu.org/lzip/lzlib.html.

Makefile.in

@@ -9,8 +9,8 @@ SHELL = /bin/sh
 CAN_RUN_INSTALLINFO = $(SHELL) -c "install-info --version" > /dev/null 2>&1
 
 objs = arg_parser.o lzip_index.o archive_reader.o common.o common_decode.o \
-       create.o create_lz.o decode.o decode_lz.o delete.o delete_lz.o \
+       compress.o create.o create_lz.o decode.o decode_lz.o delete.o \
-       exclude.o extended.o main.o
+       delete_lz.o exclude.o extended.o main.o
 
 
 .PHONY : all install install-bin install-info install-man \
@@ -35,8 +35,9 @@ arg_parser.o     : arg_parser.h
 archive_reader.o : tarlz.h lzip_index.h archive_reader.h
 common.o         : arg_parser.h tarlz.h
 common_decode.o  : arg_parser.h tarlz.h
-create.o         : arg_parser.h tarlz.h
+compress.o       : arg_parser.h tarlz.h
-create_lz.o      : arg_parser.h tarlz.h
+create.o         : arg_parser.h tarlz.h create.h
+create_lz.o      : arg_parser.h tarlz.h create.h
 decode.o         : arg_parser.h tarlz.h lzip_index.h archive_reader.h
 decode_lz.o      : arg_parser.h tarlz.h lzip_index.h archive_reader.h
 delete.o         : arg_parser.h tarlz.h lzip_index.h archive_reader.h
@@ -136,6 +137,7 @@ dist : doc
 	  $(DISTNAME)/testsuite/rbar \
 	  $(DISTNAME)/testsuite/rbaz \
 	  $(DISTNAME)/testsuite/test3.tar \
+	  $(DISTNAME)/testsuite/test3_nn.tar \
 	  $(DISTNAME)/testsuite/test3_eof[1-4].tar \
 	  $(DISTNAME)/testsuite/test3_gh[1-4].tar \
 	  $(DISTNAME)/testsuite/test3_bad[1-5].tar \
@@ -150,6 +152,7 @@ dist : doc
 	  $(DISTNAME)/testsuite/test3_eof[1-5].tar.lz \
 	  $(DISTNAME)/testsuite/test3_em[1-6].tar.lz \
 	  $(DISTNAME)/testsuite/test3_gh[1-6].tar.lz \
+	  $(DISTNAME)/testsuite/test3_nn.tar.lz \
 	  $(DISTNAME)/testsuite/test3_sm[1-4].tar.lz \
 	  $(DISTNAME)/testsuite/test3_bad[1-6].tar.lz \
 	  $(DISTNAME)/testsuite/test3_dir.tar.lz \

NEWS

@@ -1,7 +1,22 @@
-Changes in version 0.19:
+Changes in version 0.21:
 
-At verbosity level 1 or higher tarlz now prints a diagnostic for each
+Lzlib 1.12 or newer is now required to compile and run tarlz.
-unknown extended header keyword found in an archive, once per keyword.
 
-A missing '#include <sys/types.h>', which made compilation fail on some
+Members without name are now skipped when decoding except when listing. This
-systems, has been added.
+allows the reliable detection of certain format violations during parallel
+extraction with more than two threads. (Thanks to Florian Schmaus for
+reporting the problem).
+
+The new option '-z, --compress', which compresses existing POSIX tar
+archives aligning the lzip members to the tar members with choice of
+granularity, has been added. Existing compressed archives are not
+overwritten.
+
+The new option '-o, --output', which writes the compressed output to a file,
+has been added. Currently '--output' only works with '--compress'.
+
+The new option '--warn-newer', which warns during archive creation if any
+file being archived has a modification time newer than the archive creation
+time, has been added.
+
+A failure in the '--diff' test of the testsuite on OS/2 has been fixed.

archive_reader.cc

@@ -19,12 +19,7 @@
 
 #include <algorithm>
 #include <cerrno>
-#include <climits>
+#include <stdint.h>	// for lzlib.h
-#include <cstring>
-#include <string>
-#include <vector>
-#include <pthread.h>	// for tarlz.h
-#include <stdint.h>
 #include <unistd.h>
 #include <lzlib.h>
 
@@ -87,7 +82,7 @@ int Archive_reader_base::parse_records( Extended & extended,
   if( edsize <= 0 || edsize >= 1LL << 33 || bufsize >= INT_MAX )
     return 1;					// overflow or no extended data
   if( !rbuf.resize( bufsize ) ) return 1;	// extended records buffer
-  int retval = read( (uint8_t *)rbuf(), bufsize );
+  int retval = read( rbuf.u8(), bufsize );
   if( retval == 0 && !extended.parse( rbuf(), edsize, permissive ) )
     retval = 2;
   return retval;

common.cc

@@ -19,31 +19,26 @@
 
 #include <cctype>
 #include <cerrno>
-#include <climits>
 #include <cstdlib>
-#include <cstring>
-#include <string>
-#include <vector>
 #include <pthread.h>
-#include <stdint.h>
 #include <unistd.h>
 
-#include "arg_parser.h"
 #include "tarlz.h"
+#include "arg_parser.h"
 
 
 void xinit_mutex( pthread_mutex_t * const mutex )
   {
   const int errcode = pthread_mutex_init( mutex, 0 );
   if( errcode )
-    { show_error( "pthread_mutex_init", errcode ); cleanup_and_fail(); }
+    { show_error( "pthread_mutex_init", errcode ); exit_fail_mt(); }
   }
 
 void xinit_cond( pthread_cond_t * const cond )
   {
   const int errcode = pthread_cond_init( cond, 0 );
   if( errcode )
-    { show_error( "pthread_cond_init", errcode ); cleanup_and_fail(); }
+    { show_error( "pthread_cond_init", errcode ); exit_fail_mt(); }
   }
 
 
@@ -51,14 +46,14 @@ void xdestroy_mutex( pthread_mutex_t * const mutex )
   {
   const int errcode = pthread_mutex_destroy( mutex );
   if( errcode )
-    { show_error( "pthread_mutex_destroy", errcode ); cleanup_and_fail(); }
+    { show_error( "pthread_mutex_destroy", errcode ); exit_fail_mt(); }
   }
 
 void xdestroy_cond( pthread_cond_t * const cond )
   {
   const int errcode = pthread_cond_destroy( cond );
   if( errcode )
-    { show_error( "pthread_cond_destroy", errcode ); cleanup_and_fail(); }
+    { show_error( "pthread_cond_destroy", errcode ); exit_fail_mt(); }
   }
 
 
@@ -66,7 +61,7 @@ void xlock( pthread_mutex_t * const mutex )
   {
   const int errcode = pthread_mutex_lock( mutex );
   if( errcode )
-    { show_error( "pthread_mutex_lock", errcode ); cleanup_and_fail(); }
+    { show_error( "pthread_mutex_lock", errcode ); exit_fail_mt(); }
   }
 
 
@@ -74,7 +69,7 @@ void xunlock( pthread_mutex_t * const mutex )
   {
   const int errcode = pthread_mutex_unlock( mutex );
   if( errcode )
-    { show_error( "pthread_mutex_unlock", errcode ); cleanup_and_fail(); }
+    { show_error( "pthread_mutex_unlock", errcode ); exit_fail_mt(); }
   }
 
 
@@ -82,7 +77,7 @@ void xwait( pthread_cond_t * const cond, pthread_mutex_t * const mutex )
   {
   const int errcode = pthread_cond_wait( cond, mutex );
   if( errcode )
-    { show_error( "pthread_cond_wait", errcode ); cleanup_and_fail(); }
+    { show_error( "pthread_cond_wait", errcode ); exit_fail_mt(); }
   }
 
 
@@ -90,7 +85,7 @@ void xsignal( pthread_cond_t * const cond )
   {
   const int errcode = pthread_cond_signal( cond );
   if( errcode )
-    { show_error( "pthread_cond_signal", errcode ); cleanup_and_fail(); }
+    { show_error( "pthread_cond_signal", errcode ); exit_fail_mt(); }
   }
 
 
@@ -98,7 +93,7 @@ void xbroadcast( pthread_cond_t * const cond )
   {
   const int errcode = pthread_cond_broadcast( cond );
   if( errcode )
-    { show_error( "pthread_cond_broadcast", errcode ); cleanup_and_fail(); }
+    { show_error( "pthread_cond_broadcast", errcode ); exit_fail_mt(); }
   }
 
 

common_decode.cc

@@ -18,19 +18,13 @@
 #define _FILE_OFFSET_BITS 64
 
 #include <cerrno>
-#include <climits>
 #include <cstdio>
 #include <cstdlib>
-#include <cstring>
 #include <ctime>
-#include <string>
-#include <vector>
-#include <pthread.h>	// for tarlz.h
-#include <stdint.h>
 #include <sys/stat.h>
 
-#include "arg_parser.h"
 #include "tarlz.h"
+#include "arg_parser.h"
 
 
 namespace {
@@ -146,7 +140,7 @@ bool format_member_name( const Extended & extended, const Tar_header header,
     else
       offset += snprintf( rbuf() + offset, rbuf.size() - offset, " %9llu",
                           extended.file_size() );
-    for( int i = 0; i < 2; ++i )
+    for( int i = 0; i < 2; ++i )	// resize rbuf if not large enough
       {
       const int len = snprintf( rbuf() + offset, rbuf.size() - offset,
                 " %4d-%02u-%02u %02u:%02u %s%s%s\n",
@@ -186,7 +180,7 @@ bool check_skip_filename( const Cl_options & cl_opts,
                           const char * const filename )
   {
   if( Exclude::excluded( filename ) ) return true;	// skip excluded files
-  bool skip = cl_opts.filenames > 0;
+  bool skip = cl_opts.num_files > 0;
   if( skip )
     for( int i = 0; i < cl_opts.parser.arguments(); ++i )
       if( nonempty_arg( cl_opts.parser, i ) )

compress.cc

@@ -0,0 +1,378 @@
+/* Tarlz - Archiver with multimember lzip compression
+   Copyright (C) 2013-2021 Antonio Diaz Diaz.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 2 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#define _FILE_OFFSET_BITS 64
+
+#include <cerrno>
+#include <csignal>
+#include <cstdio>
+#include <cstdlib>
+#include <stdint.h>	// for lzlib.h
+#include <unistd.h>
+#include <utime.h>
+#include <sys/stat.h>
+#include <lzlib.h>
+
+#include "tarlz.h"
+#include "arg_parser.h"
+
+
+namespace {
+
+/* Variables used in signal handler context.
+   They are not declared volatile because the handler never returns. */
+std::string output_filename;
+int outfd = -1;
+bool delete_output_on_interrupt = false;
+
+
+void set_signals( void (*action)(int) )
+  {
+  std::signal( SIGHUP, action );
+  std::signal( SIGINT, action );
+  std::signal( SIGTERM, action );
+  }
+
+
+void cleanup_and_fail( const int retval )
+  {
+  set_signals( SIG_IGN );			// ignore signals
+  if( delete_output_on_interrupt )
+    {
+    delete_output_on_interrupt = false;
+    if( verbosity >= 0 )
+      std::fprintf( stderr, "%s: Deleting output file '%s', if it exists.\n",
+                    program_name, output_filename.c_str() );
+    if( outfd >= 0 ) { close( outfd ); outfd = -1; }
+    if( std::remove( output_filename.c_str() ) != 0 && errno != ENOENT )
+      show_error( "WARNING: deletion of output file (apparently) failed." );
+    }
+  std::exit( retval );
+  }
+
+
+extern "C" void signal_handler( int )
+  {
+  show_error( "Control-C or similar caught, quitting." );
+  cleanup_and_fail( 1 );
+  }
+
+
+const char * ne_output_filename()	// non-empty output file name
+  {
+  return output_filename.size() ? output_filename.c_str() : "(stdout)";
+  }
+
+
+bool check_tty_out()
+  {
+  if( isatty( outfd ) )
+    { show_file_error( ne_output_filename(),
+                       "I won't write compressed data to a terminal." );
+      return false; }
+  return true;
+  }
+
+
+// Set permissions, owner, and times.
+void close_and_set_permissions( const struct stat * const in_statsp )
+  {
+  bool warning = false;
+  if( in_statsp )
+    {
+    const mode_t mode = in_statsp->st_mode;
+    // fchown will in many cases return with EPERM, which can be safely ignored.
+    if( fchown( outfd, in_statsp->st_uid, in_statsp->st_gid ) == 0 )
+      { if( fchmod( outfd, mode ) != 0 ) warning = true; }
+    else
+      if( errno != EPERM ||
+          fchmod( outfd, mode & ~( S_ISUID | S_ISGID | S_ISVTX ) ) != 0 )
+        warning = true;
+    }
+  if( close( outfd ) != 0 )
+    {
+    show_error( "Error closing output file", errno );
+    cleanup_and_fail( 1 );
+    }
+  outfd = -1;
+  delete_output_on_interrupt = false;
+  if( in_statsp )
+    {
+    struct utimbuf t;
+    t.actime = in_statsp->st_atime;
+    t.modtime = in_statsp->st_mtime;
+    if( utime( output_filename.c_str(), &t ) != 0 ) warning = true;
+    }
+  if( warning && verbosity >= 1 )
+    show_error( "Can't change output file attributes." );
+  }
+
+
+inline void set_retval( int & retval, const int new_val )
+  { if( retval < new_val ) retval = new_val; }
+
+
+bool archive_write( const uint8_t * const buf, const long long size,
+                    LZ_Encoder * const encoder )
+  {
+  static bool flushed = true;		// avoid flushing empty lzip members
+
+  if( size <= 0 && flushed ) return true;
+  flushed = ( size <= 0 );
+  enum { obuf_size = 65536 };
+  uint8_t obuf[obuf_size];
+  long long sz = 0;
+  if( flushed ) LZ_compress_finish( encoder );	// flush encoder
+  while( sz < size || flushed )
+    {
+    if( sz < size )
+      { const int wr = LZ_compress_write( encoder, buf + sz,
+                       std::min( size - sz, (long long)max_dictionary_size ) );
+        if( wr < 0 ) internal_error( "library error (LZ_compress_write)." );
+        sz += wr; }
+    if( sz >= size && !flushed ) break;		// minimize dictionary size
+    const int rd = LZ_compress_read( encoder, obuf, obuf_size );
+    if( rd < 0 ) internal_error( "library error (LZ_compress_read)." );
+    if( rd == 0 && sz >= size ) break;
+    if( writeblock( outfd, obuf, rd ) != rd )
+      { show_file_error( ne_output_filename(), "Write error", errno );
+        return false; }
+    }
+  if( LZ_compress_finished( encoder ) == 1 &&
+      LZ_compress_restart_member( encoder, LLONG_MAX ) < 0 )
+    internal_error( "library error (LZ_compress_restart_member)." );
+  return true;
+  }
+
+
+bool tail_compress( const Cl_options & cl_opts,
+                    const int infd, Tar_header header,
+                    LZ_Encoder * const encoder )
+  {
+  if( cl_opts.solidity != solid && !archive_write( 0, 0, encoder ) )
+    return false;			// flush encoder before EOF blocks
+  int size = header_size;
+  bool zero = true;	// true until non-zero data found after EOF blocks
+  while( true )
+    {
+    if( size > 0 && !archive_write( header, size, encoder ) )
+      { close( infd ); return false; }
+    if( size < header_size ) break;			// EOF
+    size = readblock( infd, header, header_size );
+    if( errno ) return false;
+    if( zero && !block_is_zero( header, size ) )
+      { zero = false;			// flush encoder after EOF blocks
+        if( cl_opts.solidity != solid && !archive_write( 0, 0, encoder ) )
+          return false; }
+    }
+  return true;
+  }
+
+
+int compress_archive( const Cl_options & cl_opts,
+                      const std::string & input_filename,
+                      LZ_Encoder * const encoder,
+                      const bool to_stdout, const bool to_file )
+  {
+  const bool one_to_one = !to_stdout && !to_file;
+  const bool from_stdin = input_filename == "-";
+  const char * const filename = from_stdin ? "(stdin)" : input_filename.c_str();
+  const int infd = from_stdin ? STDIN_FILENO : open_instream( filename );
+  if( infd < 0 ) return 1;
+  if( one_to_one )
+    {
+    if( from_stdin ) { outfd = STDOUT_FILENO; output_filename.clear(); }
+    else
+      {
+      output_filename = input_filename + ".lz";
+      outfd = open_outstream( output_filename, true, 0, false );
+      if( outfd < 0 ) { close( infd ); return 1; }
+      delete_output_on_interrupt = true;
+      }
+    if( !check_tty_out() ) { close( infd ); return 1; }	// don't delete a tty
+    }
+  if( verbosity >= 1 ) std::fprintf( stderr, "%s\n", filename );
+
+  unsigned long long partial_data_size = 0;	// size of current block
+  Extended extended;			// metadata from extended records
+  Resizable_buffer rbuf;		// headers and extended records buffer
+  while( true )				// process one tar member per iteration
+    {
+    int total_header_size = header_size;	// size of header(s) read
+    const int rd = readblock( infd, rbuf.u8(), header_size );
+    if( rd == 0 && errno == 0 ) break;		// missing EOF blocks
+    if( rd != header_size )
+      { show_file_error( filename, "Read error", errno );
+        close( infd ); return 1; }
+
+    if( to_file && outfd < 0 )		// open outfd after verifying infd
+      {
+      outfd = open_outstream( output_filename, true, 0, false );
+      // check tty only once and don't try to delete a tty
+      if( outfd < 0 || !check_tty_out() ) { close( infd ); return 1; }
+      delete_output_on_interrupt = true;
+      }
+
+    if( !verify_ustar_chksum( rbuf.u8() ) )	// maybe EOF
+      {
+      if( block_is_zero( rbuf.u8(), header_size ) )	// first EOF block
+        { tail_compress( cl_opts, infd, rbuf.u8(), encoder ); break; }
+      show_file_error( filename, bad_hdr_msg ); close( infd ); return 2;
+      }
+
+    const Typeflag typeflag = (Typeflag)rbuf()[typeflag_o];
+    if( typeflag == tf_extended || typeflag == tf_global )
+      {
+      const long long edsize = parse_octal( rbuf.u8() + size_o, size_l );
+      const long long bufsize = round_up( edsize );
+      // overflow or no extended data
+      if( edsize <= 0 || edsize >= 1LL << 33 || bufsize >= INT_MAX )
+        { show_file_error( filename, bad_hdr_msg ); close( infd ); return 2; }
+      if( !rbuf.resize( total_header_size + bufsize ) )
+        { show_file_error( filename, mem_msg ); close( infd ); return 1; }
+      if( readblock( infd, rbuf.u8() + total_header_size, bufsize ) != bufsize )
+        { show_file_error( filename, "Read error", errno );
+          close( infd ); return 1; }
+      total_header_size += bufsize;
+      if( typeflag == tf_extended )	// do not parse global headers
+        {
+        if( !extended.parse( rbuf() + header_size, edsize, false ) )
+          { show_file_error( filename, extrec_msg ); close( infd ); return 2; }
+        // read ustar header
+        if( !rbuf.resize( total_header_size + header_size ) )
+          { show_file_error( filename, mem_msg ); close( infd ); return 1; }
+        if( readblock( infd, rbuf.u8() + total_header_size, header_size ) != header_size )
+          { show_file_error( filename, errno ? "Read error" : end_msg, errno );
+            close( infd ); return errno ? 1 : 2; }
+        if( !verify_ustar_chksum( rbuf.u8() ) )
+          { show_file_error( filename, bad_hdr_msg ); close( infd ); return 2; }
+        const Typeflag typeflag2 = (Typeflag)(rbuf() + total_header_size)[typeflag_o];
+        if( typeflag2 == tf_extended || typeflag2 == tf_global )
+          { const char * msg = ( typeflag2 == tf_global ) ? fv_msg2 : fv_msg3;
+            show_file_error( filename, msg ); close( infd ); return 2; }
+        total_header_size += header_size;
+        }
+      }
+
+    const long long file_size = round_up( extended.get_file_size_and_reset(
+                                rbuf.u8() + total_header_size - header_size ) );
+    if( cl_opts.solidity == bsolid &&
+        block_is_full( total_header_size - header_size, file_size,
+                       cl_opts.data_size, partial_data_size ) &&
+        !archive_write( 0, 0, encoder ) ) { close( infd ); return 1; }
+    if( !archive_write( rbuf.u8(), total_header_size, encoder ) )
+      { close( infd ); return 1; }
+
+    if( file_size )
+      {
+      const long long bufsize = 32 * header_size;
+      uint8_t buf[bufsize];
+      long long rest = file_size;	// file_size already rounded up
+      while( rest > 0 )
+        {
+        int size = std::min( rest, bufsize );
+        const int rd = readblock( infd, buf, size );
+        rest -= rd;
+        if( rd != size )
+          {
+          if( verbosity >= 0 )
+            std::fprintf( stderr, "'%s' ends unexpectedly at pos %llu\n",
+                          filename, file_size - rest );
+          close( infd ); return 1;
+          }
+        if( !archive_write( buf, size, encoder ) ) { close( infd ); return 1; }
+        }
+      }
+    if( cl_opts.solidity == no_solid && !archive_write( 0, 0, encoder ) )
+      { close( infd ); return 1; }	// one tar member per lzip member
+    }
+  // flush and restart encoder (for next archive)
+  if( !archive_write( 0, 0, encoder ) ) { close( infd ); return 1; }
+  const bool need_close = delete_output_on_interrupt &&
+                          ( one_to_one || ( to_file && !from_stdin ) );
+  struct stat in_stats;
+  const struct stat * const in_statsp =
+        ( need_close && fstat( infd, &in_stats ) == 0 ) ? &in_stats : 0;
+  if( close( infd ) != 0 )
+    { show_file_error( filename, "Error closing file", errno ); return 1; }
+  if( need_close ) close_and_set_permissions( in_statsp );
+  return 0;
+  }
+
+} // end namespace
+
+
+int compress( Cl_options & cl_opts )
+  {
+  if( !cl_opts.archive_name.empty() )
+    { show_file_error( cl_opts.archive_name.c_str(),
+      "Option '-f' is incompatible with '--compress'." ); return 1; }
+  if( cl_opts.num_files > 1 && cl_opts.output_filename.size() )
+    { show_file_error( cl_opts.output_filename.c_str(),
+      "Only can compress one archive when using '-o'." ); return 1; }
+  const bool to_stdout = cl_opts.output_filename == "-";
+  if( to_stdout )				// check tty only once
+    { outfd = STDOUT_FILENO; if( !check_tty_out() ) return 1; }
+  else outfd = -1;
+  const bool to_file = !to_stdout && cl_opts.output_filename.size();
+  if( to_file ) output_filename = cl_opts.output_filename;
+  if( !to_stdout && ( cl_opts.filenames_given || to_file ) )
+    set_signals( signal_handler );
+
+  const int dictionary_size = option_mapping[cl_opts.level].dictionary_size;
+  if( cl_opts.data_size <= 0 )
+    {
+    if( cl_opts.level == 0 ) cl_opts.data_size = 1 << 20;
+    else cl_opts.data_size = 2 * dictionary_size;
+    }
+  LZ_Encoder * encoder = LZ_compress_open( dictionary_size,
+                option_mapping[cl_opts.level].match_len_limit, LLONG_MAX );
+  if( !encoder || LZ_compress_errno( encoder ) != LZ_ok )
+    {
+    if( !encoder || LZ_compress_errno( encoder ) == LZ_mem_error )
+      show_error( mem_msg2 );
+    else
+      internal_error( "invalid argument to encoder." );
+    return 1;
+    }
+
+  if( !cl_opts.filenames_given )
+    return compress_archive( cl_opts, "-", encoder, to_stdout, to_file );
+  int retval = 0;
+  bool stdin_used = false;
+  for( int i = 0; i < cl_opts.parser.arguments(); ++i )
+    if( nonempty_arg( cl_opts.parser, i ) )	// skip opts, empty names
+      {
+      if( cl_opts.parser.argument( i ) == "-" )
+        { if( stdin_used ) continue; else stdin_used = true; }
+      const int tmp = compress_archive( cl_opts, cl_opts.parser.argument( i ),
+                                        encoder, to_stdout, to_file );
+      if( tmp )
+        { set_retval( retval, tmp );
+          if( delete_output_on_interrupt ) cleanup_and_fail( retval ); }
+      }
+  // flush and close encoder if needed
+  if( outfd >= 0 && archive_write( 0, 0, encoder ) &&
+      LZ_compress_close( encoder ) < 0 )
+    { show_error( "LZ_compress_close failed." ); set_retval( retval, 1 ); }
+  if( outfd >= 0 && close( outfd ) != 0 )			// to_stdout
+    {
+    show_error( "Error closing stdout", errno );
+    set_retval( retval, 1 );
+    }
+  return retval;
+  }

configure

@@ -6,7 +6,7 @@
 # to copy, distribute, and modify it.
 
 pkgname=tarlz
-pkgversion=0.19
+pkgversion=0.21
 progname=tarlz
 srctrigger=doc/${pkgname}.texi
 

create.cc

@@ -19,14 +19,10 @@
 
 #include <algorithm>
 #include <cerrno>
-#include <climits>
 #include <cstdio>
 #include <cstdlib>
-#include <cstring>
-#include <string>
-#include <vector>
 #include <pthread.h>
-#include <stdint.h>
+#include <stdint.h>	// for lzlib.h
 #include <unistd.h>
 #include <sys/stat.h>
 #if !defined __FreeBSD__ && !defined __OpenBSD__ && !defined __NetBSD__ && \
@@ -38,8 +34,12 @@
 #include <pwd.h>
 #include <lzlib.h>
 
-#include "arg_parser.h"
 #include "tarlz.h"
+#include "arg_parser.h"
+#include "create.h"
+
+
+Archive_attrs archive_attrs;	// archive attributes at time of creation
 
 
 namespace {
@@ -52,28 +52,6 @@ Resizable_buffer grbuf;				// extended header + data
 int goutfd = -1;
 int error_status = 0;
 
-class File_is_the_archive
-  {
-  dev_t archive_dev;
-  ino_t archive_ino;
-  bool initialized;
-
-public:
-  File_is_the_archive() : initialized( false ) {}
-  bool init( const int fd )
-    {
-    struct stat st;
-    if( fstat( fd, &st ) != 0 ) return false;
-    if( S_ISREG( st.st_mode ) )
-      { archive_dev = st.st_dev; archive_ino = st.st_ino; initialized = true; }
-    return true;
-    }
-  bool operator()( const struct stat & st ) const
-    {
-    return initialized && archive_dev == st.st_dev && archive_ino == st.st_ino;
-    }
-  } file_is_the_archive;
-
 
 bool option_C_after_relative_filename( const Arg_parser & parser )
   {
@@ -171,7 +149,7 @@ long long check_uncompressed_appendable( const int fd, const bool remove_eof )
       if( edsize <= 0 || edsize >= 1LL << 33 || bufsize >= INT_MAX )
         return -1;			// overflow or no extended data
       if( !rbuf.resize( bufsize ) ) return -1;
-      if( readblock( fd, (uint8_t *)rbuf(), bufsize ) != bufsize )
+      if( readblock( fd, rbuf.u8(), bufsize ) != bufsize )
         return -1;
       if( typeflag == tf_extended )
         { if( !extended.parse( rbuf(), edsize, false ) ) return -1;
@@ -229,7 +207,7 @@ bool write_extended( const Extended & extended )
   for( long long pos = 0; pos < ebsize; )    // write extended block to archive
     {
     int size = std::min( ebsize - pos, 1LL << 20 );
-    if( !archive_write( (const uint8_t *)grbuf() + pos, size ) ) return false;
+    if( !archive_write( grbuf.u8() + pos, size ) ) return false;
     pos += size;
     }
   return true;
@@ -276,8 +254,8 @@ int add_member( const char * const filename, const struct stat *,
   if( file_size && infd < 0 ) { set_error_status( 1 ); return 0; }
 
   if( encoder && gcl_opts->solidity == bsolid &&
-      block_is_full( extended, file_size, partial_data_size ) &&
+      block_is_full( extended.full_size(), file_size, gcl_opts->data_size,
-      !archive_write( 0, 0 ) ) return 1;
+                     partial_data_size ) && !archive_write( 0, 0 ) ) return 1;
 
   if( !write_extended( extended ) || !archive_write( header, header_size ) )
     return 1;
@@ -312,6 +290,9 @@ int add_member( const char * const filename, const struct stat *,
     }
   if( encoder && gcl_opts->solidity == no_solid && !archive_write( 0, 0 ) )
     return 1;
+  if( gcl_opts->warn_newer && archive_attrs.is_newer( filename ) )
+    { show_file_error( filename, "File is newer than the archive." );
+      set_error_status( 1 ); }
   if( verbosity >= 1 ) std::fprintf( stderr, "%s\n", filename );
   return 0;
   }
@@ -420,7 +401,7 @@ bool fill_headers( const char * const filename, Extended & extended,
   if( hstat( filename, &st, gcl_opts->dereference ) != 0 )
     { show_file_error( filename, "Can't stat input file", errno );
       set_error_status( 1 ); return false; }
-  if( file_is_the_archive( st ) )
+  if( archive_attrs.is_the_archive( st ) )
     { show_file_error( archive_namep, "File is the archive; not dumped." );
       return false; }
   init_tar_header( header );
@@ -516,13 +497,13 @@ bool fill_headers( const char * const filename, Extended & extended,
   }
 
 
-bool block_is_full( const Extended & extended,
+bool block_is_full( const long long extended_size,
                     const unsigned long long file_size,
+                    const unsigned long long target_size,
                     unsigned long long & partial_data_size )
   {
   const unsigned long long member_size =	// may overflow 'long long'
-    header_size + extended.full_size() + round_up( file_size );
+    header_size + extended_size + round_up( file_size );
-  const unsigned long long target_size = gcl_opts->data_size;
   if( partial_data_size >= target_size ||
       ( partial_data_size >= min_data_size &&
         partial_data_size + member_size / 2 > target_size ) )
@@ -550,7 +531,7 @@ int final_exit_status( int retval, const bool show_msg )
   return retval;
   }
 
-unsigned ustar_chksum( const uint8_t * const header )
+unsigned ustar_chksum( const Tar_header header )
   {
   unsigned chksum = chksum_l * 0x20;	// treat chksum field as spaces
   for( int i = 0; i < chksum_o; ++i ) chksum += header[i];
@@ -559,7 +540,7 @@ unsigned ustar_chksum( const uint8_t * const header )
   }
 
 
-bool verify_ustar_chksum( const uint8_t * const header )
+bool verify_ustar_chksum( const Tar_header header )
   { return ( verify_ustar_magic( header ) &&
     ustar_chksum( header ) == parse_octal( header + chksum_o, chksum_l ) ); }
 
@@ -575,14 +556,14 @@ bool has_lz_ext( const std::string & name )
 
 int concatenate( const Cl_options & cl_opts )
   {
-  if( cl_opts.filenames <= 0 )
+  if( cl_opts.num_files <= 0 )
     { if( verbosity >= 1 ) show_error( "Nothing to concatenate." ); return 0; }
   const bool to_stdout = cl_opts.archive_name.empty();
   archive_namep = to_stdout ? "(stdout)" : cl_opts.archive_name.c_str();
   const int outfd =
     to_stdout ? STDOUT_FILENO : open_outstream( cl_opts.archive_name, false );
   if( outfd < 0 ) return 1;
-  if( !to_stdout && !file_is_the_archive.init( outfd ) )
+  if( !to_stdout && !archive_attrs.init( outfd ) )
     { show_file_error( archive_namep, "Can't stat", errno ); return 1; }
   int compressed;					// tri-state bool
   if( to_stdout ) compressed = -1;			// unknown
@@ -613,7 +594,7 @@ int concatenate( const Cl_options & cl_opts )
     const int infd = open_instream( filename );
     if( infd < 0 ) { retval = 1; break; }
     struct stat st;
-    if( !to_stdout && fstat( infd, &st ) == 0 && file_is_the_archive( st ) )
+    if( !to_stdout && fstat( infd, &st ) == 0 && archive_attrs.is_the_archive( st ) )
       { show_file_error( filename, "File is the archive; not concatenated." );
         close( infd ); continue; }
     long long size;
@@ -649,23 +630,6 @@ int concatenate( const Cl_options & cl_opts )
 
 int encode( Cl_options & cl_opts )
   {
-  struct Lzma_options
-    {
-    int dictionary_size;		// 4 KiB .. 512 MiB
-    int match_len_limit;		// 5 .. 273
-    };
-  const Lzma_options option_mapping[] =
-    {
-    {   65535,  16 },		// -0
-    { 1 << 20,   5 },		// -1
-    { 3 << 19,   6 },		// -2
-    { 1 << 21,   8 },		// -3
-    { 3 << 20,  12 },		// -4
-    { 1 << 22,  20 },		// -5
-    { 1 << 23,  36 },		// -6
-    { 1 << 24,  68 },		// -7
-    { 3 << 23, 132 },		// -8
-    { 1 << 25, 273 } };		// -9
   const bool compressed = ( cl_opts.level >= 0 && cl_opts.level <= 9 );
   const bool to_stdout = cl_opts.archive_name.empty();
   archive_namep = to_stdout ? "(stdout)" : cl_opts.archive_name.c_str();
@@ -676,7 +640,7 @@ int encode( Cl_options & cl_opts )
       "Uncompressed mode incompatible with .lz extension." ); return 2; }
 
   const bool append = cl_opts.program_mode == m_append;
-  if( cl_opts.filenames <= 0 )
+  if( cl_opts.num_files <= 0 )
     {
     if( !append && !to_stdout )			// create archive
       { show_error( "Cowardly refusing to create an empty archive.", 0, true );
@@ -701,7 +665,7 @@ int encode( Cl_options & cl_opts )
         "This does not look like an appendable tar archive." ); return 2; }
     }
 
-  if( !file_is_the_archive.init( goutfd ) )
+  if( !archive_attrs.init( goutfd ) )
     { show_file_error( archive_namep, "Can't stat", errno ); return 1; }
 
   if( compressed )

create.h

@@ -0,0 +1,45 @@
+/* Tarlz - Archiver with multimember lzip compression
+   Copyright (C) 2013-2021 Antonio Diaz Diaz.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 2 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+class Archive_attrs
+  {
+  struct stat ast;		// archive attributes at time of init
+  bool initialized;
+  bool isreg;
+
+public:
+  Archive_attrs() : initialized( false ), isreg( false ) {}
+  bool init( const int fd )
+    {
+    if( fstat( fd, &ast ) != 0 ) return false;
+    if( S_ISREG( ast.st_mode ) ) isreg = true;
+    initialized = true;
+    return true;
+    }
+  bool is_the_archive( const struct stat & st ) const
+    { return isreg && st.st_dev == ast.st_dev && st.st_ino == ast.st_ino; }
+  bool is_newer( const struct stat & st ) const
+    { return initialized && st.st_mtime > ast.st_mtime; }
+  bool is_newer( const char * const filename ) const
+    {
+    if( !initialized ) return false;
+    struct stat st;
+    return lstat( filename, &st ) != 0 || st.st_mtime > ast.st_mtime;
+    }
+  };
+
+extern Archive_attrs archive_attrs;

create_lz.cc

@@ -19,22 +19,19 @@
 
 #include <algorithm>
 #include <cerrno>
-#include <climits>
 #include <cstdio>
 #include <cstdlib>
-#include <cstring>
 #include <queue>
-#include <string>
-#include <vector>
 #include <pthread.h>
-#include <stdint.h>
+#include <stdint.h>	// for lzlib.h
 #include <unistd.h>
 #include <sys/stat.h>
 #include <ftw.h>
 #include <lzlib.h>
 
-#include "arg_parser.h"
 #include "tarlz.h"
+#include "arg_parser.h"
+#include "create.h"
 
 
 namespace {
@@ -272,7 +269,8 @@ int add_member_lz( const char * const filename, const struct stat *,
     { delete[] header; delete extended; return 0; }
 
   if( gcl_opts->solidity == bsolid &&
-      block_is_full( *extended, file_size, partial_data_size ) )
+      block_is_full( extended->full_size(), file_size, gcl_opts->data_size,
+                     partial_data_size ) )
     courierp->receive_packet( new Ipacket );		// end of group
 
   courierp->receive_packet( new Ipacket( filename, file_size, extended, header ) );
@@ -307,7 +305,7 @@ extern "C" void * grouper( void * arg )
     const char * filename = arg.c_str();
     if( code == 'C' && chdir( filename ) != 0 )
       { show_file_error( filename, "Error changing working directory", errno );
-        cleanup_and_fail(); }
+        exit_fail_mt(); }
     if( code ) continue;				// skip options
     if( cl_opts.parser.argument( i ).empty() ) continue; // skip empty names
     std::string deslashed;		// arg without trailing slashes
@@ -322,7 +320,7 @@ extern "C" void * grouper( void * arg )
         set_error_status( 1 ); }
     else if( nftw( filename, add_member_lz, 16,
                    cl_opts.dereference ? 0 : FTW_PHYS ) != 0 )
-      cleanup_and_fail();			// write error or OOM
+      exit_fail_mt();			// write error or OOM
     else if( cl_opts.solidity == dsolid )	// end of group
       courier.receive_packet( new Ipacket );
     }
@@ -363,7 +361,7 @@ void loop_encode( const uint8_t * const ibuf, const int isize,
       if( verbosity >= 0 )
         std::fprintf( stderr, "LZ_compress_read error: %s\n",
                       LZ_strerror( LZ_compress_errno( encoder ) ) );
-      cleanup_and_fail();
+      exit_fail_mt();
       }
     opos += rd;
     // obuf is full or last opacket in lzip member
@@ -373,11 +371,11 @@ void loop_encode( const uint8_t * const ibuf, const int isize,
         internal_error( "opacket size exceeded in worker." );
       courier.collect_packet( new Opacket( obuf, opos ), worker_id );
       opos = 0; obuf = new( std::nothrow ) uint8_t[max_packet_size];
-      if( !obuf ) { show_error( mem_msg2 ); cleanup_and_fail(); }
+      if( !obuf ) { show_error( mem_msg2 ); exit_fail_mt(); }
       if( LZ_compress_finished( encoder ) == 1 )
         {
         if( LZ_compress_restart_member( encoder, LLONG_MAX ) >= 0 ) break;
-        show_error( "LZ_compress_restart_member failed." ); cleanup_and_fail();
+        show_error( "LZ_compress_restart_member failed." ); exit_fail_mt();
         }
       }
     }
@@ -409,7 +407,7 @@ extern "C" void * cworker( void * arg )
   LZ_Encoder * encoder = 0;
   uint8_t * data = 0;
   Resizable_buffer rbuf;			// extended header + data
-  if( !rbuf.size() ) { show_error( mem_msg2 ); cleanup_and_fail(); }
+  if( !rbuf.size() ) { show_error( mem_msg2 ); exit_fail_mt(); }
 
   int opos = 0;
   bool flushed = true;		// avoid producing empty lzip members
@@ -425,8 +423,9 @@ extern "C" void * cworker( void * arg )
       flushed = true; delete ipacket; continue;
       }
 
+    const char * const filename = ipacket->filename.c_str();
     const int infd =
-      ipacket->file_size ? open_instream( ipacket->filename.c_str() ) : -1;
+      ipacket->file_size ? open_instream( filename ) : -1;
     if( ipacket->file_size && infd < 0 )	// can't read file data
       { delete[] ipacket->header; delete ipacket->extended; delete ipacket;
         set_error_status( 1 ); continue; }	// skip file
@@ -442,7 +441,7 @@ extern "C" void * cworker( void * arg )
           show_error( mem_msg2 );
         else
           internal_error( "invalid argument to encoder." );
-        cleanup_and_fail();
+        exit_fail_mt();
         }
       }
 
@@ -450,13 +449,12 @@ extern "C" void * cworker( void * arg )
       {
       const long long ebsize = ipacket->extended->format_block( rbuf );
       if( ebsize < 0 )
-        { show_error( "Error formatting extended records." ); cleanup_and_fail(); }
+        { show_error( "Error formatting extended records." ); exit_fail_mt(); }
       /* Limit the size of the extended block to INT_MAX - 1 so that it can
          be fed to lzlib as one buffer. */
       if( ebsize >= INT_MAX )
-        { show_error( "Extended records size >= INT_MAX." ); cleanup_and_fail(); }
+        { show_error( "Extended records size >= INT_MAX." ); exit_fail_mt(); }
-      loop_encode( (const uint8_t *)rbuf(), ebsize, data, opos, courier,
+      loop_encode( rbuf.u8(), ebsize, data, opos, courier, encoder, worker_id );
-                   encoder, worker_id );
       }
     // compress ustar header
     loop_encode( ipacket->header, header_size, data, opos, courier,
@@ -477,8 +475,8 @@ extern "C" void * cworker( void * arg )
           {
           if( verbosity >= 0 )
             std::fprintf( stderr, "File '%s' ends unexpectedly at pos %llu\n",
-                          ipacket->filename.c_str(), ipacket->file_size - rest );
+                          filename, ipacket->file_size - rest );
-          close( infd ); cleanup_and_fail();
+          close( infd ); exit_fail_mt();
           }
         if( rest == 0 )				// last read
           {
@@ -491,14 +489,17 @@ extern "C" void * cworker( void * arg )
         loop_encode( buf, size, data, opos, courier, encoder, worker_id );
         }
       if( close( infd ) != 0 )
-        { show_file_error( ipacket->filename.c_str(), "Error closing file", errno );
+        { show_file_error( filename, "Error closing file", errno );
-          cleanup_and_fail(); }
+          exit_fail_mt(); }
       }
+    if( gcl_opts->warn_newer && archive_attrs.is_newer( filename ) )
+      { show_file_error( filename, "File is newer than the archive." );
+        set_error_status( 1 ); }
     delete ipacket;
     }
   if( data ) delete[] data;
   if( encoder && LZ_compress_close( encoder ) < 0 )
-    { show_error( "LZ_compress_close failed." ); cleanup_and_fail(); }
+    { show_error( "LZ_compress_close failed." ); exit_fail_mt(); }
   return 0;
   }
 
@@ -514,7 +515,7 @@ void muxer( Packet_courier & courier, const int outfd )
     if( !opacket ) break;	// queue is empty. all workers exited
 
     if( !writeblock_wrapper( outfd, opacket->data, opacket->size ) )
-      cleanup_and_fail();
+      exit_fail_mt();
     delete[] opacket->data;
     delete opacket;
     }
@@ -546,12 +547,12 @@ int encode_lz( const Cl_options & cl_opts, const char * const archive_namep,
   pthread_t grouper_thread;
   int errcode = pthread_create( &grouper_thread, 0, grouper, &grouper_arg );
   if( errcode )
-    { show_error( "Can't create grouper thread", errcode ); cleanup_and_fail(); }
+    { show_error( "Can't create grouper thread", errcode ); exit_fail_mt(); }
 
   Worker_arg * worker_args = new( std::nothrow ) Worker_arg[num_workers];
   pthread_t * worker_threads = new( std::nothrow ) pthread_t[num_workers];
   if( !worker_args || !worker_threads )
-    { show_error( mem_msg ); cleanup_and_fail(); }
+    { show_error( mem_msg ); exit_fail_mt(); }
   for( int i = 0; i < num_workers; ++i )
     {
     worker_args[i].courier = &courier;
@@ -560,7 +561,7 @@ int encode_lz( const Cl_options & cl_opts, const char * const archive_namep,
     worker_args[i].worker_id = i;
     errcode = pthread_create( &worker_threads[i], 0, cworker, &worker_args[i] );
     if( errcode )
-      { show_error( "Can't create worker threads", errcode ); cleanup_and_fail(); }
+      { show_error( "Can't create worker threads", errcode ); exit_fail_mt(); }
     }
 
   muxer( courier, outfd );
@@ -569,14 +570,14 @@ int encode_lz( const Cl_options & cl_opts, const char * const archive_namep,
     {
     errcode = pthread_join( worker_threads[i], 0 );
     if( errcode )
-      { show_error( "Can't join worker threads", errcode ); cleanup_and_fail(); }
+      { show_error( "Can't join worker threads", errcode ); exit_fail_mt(); }
     }
   delete[] worker_threads;
   delete[] worker_args;
 
   errcode = pthread_join( grouper_thread, 0 );
   if( errcode )
-    { show_error( "Can't join grouper thread", errcode ); cleanup_and_fail(); }
+    { show_error( "Can't join grouper thread", errcode ); exit_fail_mt(); }
 
   // write End-Of-Archive records
   int retval = !write_eof_records( outfd, true );

decode.cc

@@ -20,14 +20,9 @@
 #include <algorithm>
 #include <cctype>
 #include <cerrno>
-#include <climits>
 #include <cstdio>
 #include <cstdlib>
-#include <cstring>
+#include <stdint.h>	// for lzlib.h
-#include <string>
-#include <vector>
-#include <pthread.h>	// for tarlz.h
-#include <stdint.h>
 #include <unistd.h>
 #include <utime.h>
 #include <sys/stat.h>
@@ -37,8 +32,8 @@
 #endif
 #include <lzlib.h>
 
-#include "arg_parser.h"
 #include "tarlz.h"
+#include "arg_parser.h"
 #include "lzip_index.h"
 #include "archive_reader.h"
 
@@ -454,10 +449,12 @@ int decode( const Cl_options & cl_opts )
 
     extended.fill_from_ustar( header );		// copy metadata from header
 
+    // members without name are skipped except when listing
     if( check_skip_filename( cl_opts, name_pending, extended.path().c_str() ) )
       retval = skip_member( ar, extended );
     else if( cl_opts.program_mode == m_list )
       retval = list_member( ar, extended, header );
+    else if( extended.path().empty() ) retval = skip_member( ar, extended );
     else if( cl_opts.program_mode == m_diff )
       retval = compare_member( cl_opts, ar, extended, header );
     else retval = extract_member( cl_opts, ar, extended, header );

decode_lz.cc

@@ -19,15 +19,11 @@
 
 #include <algorithm>
 #include <cerrno>
-#include <climits>
 #include <cstdio>
 #include <cstdlib>
-#include <cstring>
 #include <queue>
-#include <string>
-#include <vector>
 #include <pthread.h>
-#include <stdint.h>
+#include <stdint.h>	// for lzlib.h
 #include <unistd.h>
 #include <utime.h>
 #include <sys/stat.h>
@@ -37,8 +33,8 @@
 #endif
 #include <lzlib.h>
 
-#include "arg_parser.h"
 #include "tarlz.h"
+#include "arg_parser.h"
 #include "lzip_index.h"
 #include "archive_reader.h"
 
@@ -610,11 +606,16 @@ extern "C" void * dworker( void * arg )
 
       extended.fill_from_ustar( header );	// copy metadata from header
 
+      /* Skip members with an empty name in the ustar header. If there is an
+         extended header in a previous lzip member, its worker will request
+         mastership. Else the ustar-only unnamed member will be ignored. */
       const char * msg;
       if( check_skip_filename( cl_opts, name_pending, extended.path().c_str() ) )
         msg = skip_member_lz( ar, courier, extended, i, worker_id );
       else if( cl_opts.program_mode == m_list )
         msg = list_member_lz( ar, courier, extended, header, rbuf, i, worker_id );
+      else if( extended.path().empty() )
+        msg = skip_member_lz( ar, courier, extended, i, worker_id );
       else if( cl_opts.program_mode == m_diff )
         msg = compare_member_lz( cl_opts, ar, courier, extended, header,
                                  rbuf, i, worker_id );
@@ -657,7 +658,7 @@ void muxer( const char * const archive_namep, Packet_courier & courier )
     }
   if( !error && !courier.eof_found() )		// no worker found EOF blocks
     { show_file_error( archive_namep, end_msg ); error = true; }
-  if( error ) cleanup_and_fail( 2 );
+  if( error ) exit_fail_mt( 2 );
   }
 
 } // end namespace
@@ -693,7 +694,7 @@ int decode_lz( const Cl_options & cl_opts, const Archive_descriptor & ad,
     const int errcode =
       pthread_create( &worker_threads[i], 0, dworker, &worker_args[i] );
     if( errcode )
-      { show_error( "Can't create worker threads", errcode ); cleanup_and_fail(); }
+      { show_error( "Can't create worker threads", errcode ); exit_fail_mt(); }
     }
 
   muxer( ad.namep, courier );
@@ -702,7 +703,7 @@ int decode_lz( const Cl_options & cl_opts, const Archive_descriptor & ad,
     {
     const int errcode = pthread_join( worker_threads[i], 0 );
     if( errcode )
-      { show_error( "Can't join worker threads", errcode ); cleanup_and_fail(); }
+      { show_error( "Can't join worker threads", errcode ); exit_fail_mt(); }
     }
   delete[] worker_threads;
   delete[] worker_args;

delete.cc

@@ -19,19 +19,14 @@
 
 #include <cctype>
 #include <cerrno>
-#include <climits>
 #include <cstdio>
 #include <cstdlib>
-#include <cstring>
+#include <stdint.h>	// for lzlib.h
-#include <string>
-#include <vector>
-#include <pthread.h>	// for tarlz.h
-#include <stdint.h>
 #include <unistd.h>
 #include <lzlib.h>
 
-#include "arg_parser.h"
 #include "tarlz.h"
+#include "arg_parser.h"
 #include "lzip_index.h"
 #include "archive_reader.h"
 
@@ -85,7 +80,7 @@ int tail_copy( const Arg_parser & parser, const Archive_descriptor & ad,
 */
 int delete_members( const Cl_options & cl_opts )
   {
-  if( cl_opts.filenames <= 0 )
+  if( cl_opts.num_files <= 0 )
     { if( verbosity >= 1 ) show_error( "Nothing to delete." ); return 0; }
   if( cl_opts.archive_name.empty() )
     { show_error( "Deleting from stdin not implemented yet." ); return 1; }

delete_lz.cc

@@ -19,19 +19,14 @@
 
 #include <cctype>
 #include <cerrno>
-#include <climits>
 #include <cstdio>
 #include <cstdlib>
-#include <cstring>
+#include <stdint.h>	// for lzlib.h
-#include <string>
-#include <vector>
-#include <pthread.h>	// for tarlz.h
-#include <stdint.h>
 #include <unistd.h>
 #include <lzlib.h>
 
-#include "arg_parser.h"
 #include "tarlz.h"
+#include "arg_parser.h"
 #include "lzip_index.h"
 #include "archive_reader.h"
 

doc/tarlz.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.16.
-.TH TARLZ "1" "January 2021" "tarlz 0.19" "User Commands"
+.TH TARLZ "1" "June 2021" "tarlz 0.21" "User Commands"
 .SH NAME
 tarlz \- creates tar archives with multimember lzip compression
 .SH SYNOPSIS
@@ -72,6 +72,9 @@ use <date> as mtime for files added to archive
 \fB\-n\fR, \fB\-\-threads=\fR<n>
 set number of (de)compression threads [2]
 .TP
+\fB\-o\fR, \fB\-\-output=\fR<file>
+compress to <file>
+.TP
 \fB\-p\fR, \fB\-\-preserve\-permissions\fR
 don't subtract the umask on extraction
 .TP
@@ -90,6 +93,9 @@ verbosely list files processed
 \fB\-x\fR, \fB\-\-extract\fR
 extract files/directories from an archive
 .TP
+\fB\-z\fR, \fB\-\-compress\fR
+compress existing POSIX tar archives
+.TP
 \fB\-0\fR .. \fB\-9\fR
 set compression level [default 6]
 .TP
@@ -131,10 +137,13 @@ number of 1 MiB output packets buffered [64]
 .TP
 \fB\-\-check\-lib\fR
 compare version of lzlib.h with liblz.{a,so}
+.TP
+\fB\-\-warn\-newer\fR
+warn if any file is newer than the archive
 .PP
 Exit status: 0 for a normal exit, 1 for environmental problems (file not
 found, files differ, invalid flags, I/O errors, etc), 2 to indicate a
-corrupt or invalid input file, 3 for an internal consistency error (eg, bug)
+corrupt or invalid input file, 3 for an internal consistency error (e.g. bug)
 which caused tarlz to panic.
 .SH "REPORTING BUGS"
 Report bugs to lzip\-bug@nongnu.org

doc/tarlz.info

@@ -11,7 +11,7 @@ File: tarlz.info,  Node: Top,  Next: Introduction,  Up: (dir)
 Tarlz Manual
 ************
 
-This manual is for Tarlz (version 0.19, 8 January 2021).
+This manual is for Tarlz (version 0.21, 14 June 2021).
 
 * Menu:
 
@@ -102,7 +102,7 @@ The format for running tarlz is:
      tarlz [OPTIONS] [FILES]
 
 All operations except '--concatenate' operate on whole trees if any FILE is
-a directory.
+a directory. Tarlz overwrites output files without warning.
 
    On archive creation or appending tarlz archives the files specified, but
 removes from member names any leading and trailing slashes and any file name
@@ -119,7 +119,7 @@ member names in the archive or given in the command line, so that
 
    If several compression levels or '--*solid' options are given, the last
 setting is used. For example '-9 --solid --uncompressed -1' is equivalent
-to '-1 --solid'
+to '-1 --solid'.
 
    tarlz supports the following options: *Note Argument syntax:
 (arg_parser)Argument syntax.
@@ -134,16 +134,17 @@ to '-1 --solid'
 
 '-A'
 '--concatenate'
-     Append one or more archives to the end of an archive. All the archives
+     Append one or more archives to the end of an archive. If no archive is
-     involved must be regular (seekable) files, and must be either all
+     specified with the option '-f', the input archives are concatenated to
-     compressed or all uncompressed. Compressed and uncompressed archives
+     standard output. All the archives involved must be regular (seekable)
-     can't be mixed. Compressed archives must be multimember lzip files
+     files, and must be either all compressed or all uncompressed.
-     with the two end-of-file blocks plus any zero padding contained in the
+     Compressed and uncompressed archives can't be mixed. Compressed
-     last lzip member of each archive. The intermediate end-of-file blocks
+     archives must be multimember lzip files with the two end-of-file
-     are removed as each new archive is concatenated. If the archive is
+     blocks plus any zero padding contained in the last lzip member of each
-     uncompressed, tarlz parses and skips tar headers until it finds the
+     archive. The intermediate end-of-file blocks are removed as each new
-     end-of-file blocks. Exit with status 0 without modifying the archive
+     archive is concatenated. If the archive is uncompressed, tarlz parses
-     if no FILES have been specified.
+     and skips tar headers until it finds the end-of-file blocks. Exit with
+     status 0 without modifying the archive if no FILES have been specified.
 
 '-B BYTES'
 '--data-size=BYTES'
@@ -249,6 +250,12 @@ to '-1 --solid'
      and during decompression to the number of lzip members in the tar.lz
      archive, which you can find by running 'lzip -lv archive.tar.lz'.
 
+'-o FILE'
+'--output=FILE'
+     Write the compressed output to FILE. '-o -' writes the compressed
+     output to standard output. Currently '--output' only works with
+     '--compress'.
+
 '-p'
 '--preserve-permissions'
      On extraction, set file permissions as they appear in the archive.
@@ -295,11 +302,35 @@ to '-1 --solid'
      example, extracting a link over a directory will usually fail.
      (Principle of least surprise).
 
+'-z'
+'--compress'
+     Compress existing POSIX tar archives aligning the lzip members to the
+     tar members with choice of granularity (--bsolid by default, --dsolid
+     works like --asolid). The input archives are kept unchanged. Existing
+     compressed archives are not overwritten. A hyphen '-' used as the name
+     of an input archive reads from standard input and writes to standard
+     output (unless the option '--output' is used). Tarlz can be used as
+     compressor for GNU tar using a command like
+     'tar -c -Hustar foo | tarlz -z -o foo.tar.lz'. Note that tarlz only
+     works reliably on archives without global headers, or with global
+     headers whose content can be ignored.
+
+     The compression is reversible, including any garbage present after the
+     EOF blocks. Tarlz stops parsing after the first EOF block is found,
+     and then compresses the rest of the archive. Unless solid compression
+     is requested, the EOF blocks are compressed in a lzip member separated
+     from the preceding members and from any non-zero garbage following the
+     EOF blocks. '--compress' implies plzip argument style, not tar style.
+     Each input archive is compressed to a file with the extension '.lz'
+     added unless the option '--output' is used. When '--output' is used,
+     only one input archive can be specified. '-f' can't be used with
+     '--compress'.
+
 '-0 .. -9'
-     Set the compression level for '--create' and '--append'. The default
+     Set the compression level for '--create', '--append', and
-     compression level is '-6'. Like lzip, tarlz also minimizes the
+     '--compress'. The default compression level is '-6'. Like lzip, tarlz
-     dictionary size of the lzip members it creates, reducing the amount of
+     also minimizes the dictionary size of the lzip members it creates,
-     memory required for decompression.
+     reducing the amount of memory required for decompression.
 
      Level   Dictionary size   Match length limit
      -0      64 KiB            16 bytes
@@ -408,11 +439,20 @@ to '-1 --solid'
      the version of lzlib being used and the value of 'LZ_API_VERSION' (if
      defined). *Note Library version: (lzlib)Library version.
 
+'--warn-newer'
+     During archive creation, warn if any file being archived has a
+     modification time newer than the archive creation time. This option
+     may slow archive creation somewhat because it makes an extra call to
+     'stat' after archiving each file, but it guarantees that file contents
+     were not modified during the creation of the archive. Note that the
+     file must be at least one second newer than the archive for it to be
+     detected as newer.
+
 
    Exit status: 0 for a normal exit, 1 for environmental problems (file not
 found, files differ, invalid flags, I/O errors, etc), 2 to indicate a
-corrupt or invalid input file, 3 for an internal consistency error (eg, bug)
+corrupt or invalid input file, 3 for an internal consistency error (e.g.
-which caused tarlz to panic.
+bug) which caused tarlz to panic.
 
 
 File: tarlz.info,  Node: Portable character set,  Next: File format,  Prev: Invoking tarlz,  Up: Top
@@ -430,7 +470,9 @@ The set of characters from which portable file names are constructed.
 characters, respectively.
 
    File names are identifiers. Therefore, archiving works better when file
-names use only the portable character set without spaces added.
+names use only the portable character set without spaces added. Unicode is
+for human consumption. It should be avoided in computing environments,
+specially in file names. *Note why not Unicode: (moe)why not Unicode.
 
 
 File: tarlz.info,  Node: File format,  Next: Amendments to pax format,  Prev: Portable character set,  Up: Top
@@ -718,8 +760,9 @@ integrity checking of lzip may not be able to detect the corruption before
 the metadata has been used, for example, to create a new file in the wrong
 place.
 
-   Because of the above, tarlz protects the extended records with a CRC in a
+   Because of the above, tarlz protects the extended records with a Cyclic
-way compatible with standard tar tools. *Note key_crc32::.
+Redundancy Check (CRC) in a way compatible with standard tar tools. *Note
+key_crc32::.
 
 
 5.2 Remove flawed backward compatibility
@@ -740,7 +783,9 @@ to extract a file with a truncated file name.
 zeroed except size, chksum, typeflag, magic and version. This prevents old
 tar programs from extracting the extended records as a file in the wrong
 place. Tarlz also sets to zero those fields of the ustar header overridden
-by extended records.
+by extended records. Finally, tarlz skips members without name when decoding
+except when listing. This is needed to detect certain format violations
+during parallel extraction.
 
    If an extended header is required for any reason (for example a file size
 larger than 8 GiB or a link name longer than 100 bytes), tarlz moves the
@@ -860,11 +905,20 @@ File: tarlz.info,  Node: Multi-threaded decoding,  Next: Minimum archive sizes,
 7 Limitations of parallel tar decoding
 **************************************
 
-Safely decoding an arbitrary tar archive in parallel is impossible. For
+Safely decoding an arbitrary tar archive in parallel is only possible if one
-example, if a tar archive containing another tar archive is decoded starting
+decodes the headers sequentially first. For example, if a tar archive
-from some position other than the beginning, there is no way to know if the
+containing another tar archive is decoded starting from some position other
-first header found there belongs to the outer tar archive or to the inner
+than the beginning, there is no way to know if the first header found there
-tar archive. Tar is a format inherently serial; it was designed for tapes.
+belongs to the outer tar archive or to the inner tar archive. Tar is a
+format inherently serial; it was designed for tapes.
+
+   The pax format is even more serial than the ustar format. Two headers
+need to be decoded sequentially for each file. The extended header may even
+need parsing to reveal something as basic as file size. If a thread decodes
+the ustar header skipping the preceding extended header, it may extract a
+file of incorrect size at the wrong place. Moreover, a pax archive with
+global headers can't be decoded in parallel because each thread can't know
+about the global headers decoded by other threads.
 
    In the case of compressed tar archives, the start of each compressed
 block determines one point through which the tar archive can be decoded in
@@ -1026,6 +1080,14 @@ Example 8: Copy the contents of directory 'sourcedir' to the directory
 
      tarlz -C sourcedir -c . | tarlz -C destdir -x
 
+
+Example 9: Compress the existing POSIX archive 'archive.tar' and write the
+output to 'archive.tar.lz'. Compress each member individually for maximum
+availability. (If one member in the compressed archive gets damaged, the
+other members can still be extracted).
+
+     tarlz -z --no-solid archive.tar
+
 
 File: tarlz.info,  Node: Problems,  Next: Concept index,  Prev: Examples,  Up: Top
 
@@ -1069,22 +1131,22 @@ Concept index
 
 Tag Table:
 Node: Top223
-Node: Introduction1214
+Node: Introduction1212
-Node: Invoking tarlz4022
+Node: Invoking tarlz4020
-Ref: --data-size6233
+Ref: --data-size6389
-Ref: --bsolid14593
+Ref: --bsolid16341
-Node: Portable character set18852
+Node: Portable character set21079
-Node: File format19495
+Node: File format21874
-Ref: key_crc3224420
+Ref: key_crc3226799
-Node: Amendments to pax format30021
+Node: Amendments to pax format32400
-Ref: crc3230685
+Ref: crc3233064
-Ref: flawed-compat31970
+Ref: flawed-compat34375
-Node: Program design34615
+Node: Program design37176
-Node: Multi-threaded decoding38540
+Node: Multi-threaded decoding41101
-Node: Minimum archive sizes42482
+Node: Minimum archive sizes45590
-Node: Examples44620
+Node: Examples47728
-Node: Problems46335
+Node: Problems49744
-Node: Concept index46863
+Node: Concept index50272
 
 End Tag Table
 

doc/tarlz.texi

@@ -6,8 +6,8 @@
 @finalout
 @c %**end of header
 
-@set UPDATED 8 January 2021
+@set UPDATED 14 June 2021
-@set VERSION 0.19
+@set VERSION 0.21
 
 @dircategory Data Compression
 @direntry
@@ -138,7 +138,7 @@ tarlz [@var{options}] [@var{files}]
 
 @noindent
 All operations except @samp{--concatenate} operate on whole trees if any
-@var{file} is a directory.
+@var{file} is a directory. Tarlz overwrites output files without warning.
 
 On archive creation or appending tarlz archives the files specified, but
 removes from member names any leading and trailing slashes and any file name
@@ -155,7 +155,7 @@ member names in the archive or given in the command line, so that
 
 If several compression levels or @samp{--*solid} options are given, the last
 setting is used. For example @w{@samp{-9 --solid --uncompressed -1}} is
-equivalent to @samp{-1 --solid}
+equivalent to @w{@samp{-1 --solid}}.
 
 tarlz supports the following
 @uref{http://www.nongnu.org/arg-parser/manual/arg_parser_manual.html#Argument-syntax,,options}:
@@ -174,15 +174,17 @@ This version number should be included in all bug reports.
 
 @item -A
 @itemx --concatenate
-Append one or more archives to the end of an archive. All the archives
+Append one or more archives to the end of an archive. If no archive is
-involved must be regular (seekable) files, and must be either all compressed
+specified with the option @samp{-f}, the input archives are concatenated to
-or all uncompressed. Compressed and uncompressed archives can't be mixed.
+standard output. All the archives involved must be regular (seekable) files,
-Compressed archives must be multimember lzip files with the two end-of-file
+and must be either all compressed or all uncompressed. Compressed and
-blocks plus any zero padding contained in the last lzip member of each
+uncompressed archives can't be mixed. Compressed archives must be
-archive. The intermediate end-of-file blocks are removed as each new archive
+multimember lzip files with the two end-of-file blocks plus any zero padding
-is concatenated. If the archive is uncompressed, tarlz parses and skips tar
+contained in the last lzip member of each archive. The intermediate
-headers until it finds the end-of-file blocks. Exit with status 0 without
+end-of-file blocks are removed as each new archive is concatenated. If the
-modifying the archive if no @var{files} have been specified.
+archive is uncompressed, tarlz parses and skips tar headers until it finds
+the end-of-file blocks. Exit with status 0 without modifying the archive if
+no @var{files} have been specified.
 
 @anchor{--data-size}
 @item -B @var{bytes}
@@ -285,6 +287,12 @@ Note that the number of usable threads is limited during compression to
 and during decompression to the number of lzip members in the tar.lz
 archive, which you can find by running @w{@samp{lzip -lv archive.tar.lz}}.
 
+@item -o @var{file}
+@itemx --output=@var{file}
+Write the compressed output to @var{file}. @w{@samp{-o -}} writes the
+compressed output to standard output. Currently @samp{--output} only works
+with @samp{--compress}.
+
 @item -p
 @itemx --preserve-permissions
 On extraction, set file permissions as they appear in the archive. This is
@@ -331,11 +339,34 @@ special effort to extract a file over an incompatible type of file. For
 example, extracting a link over a directory will usually fail. (Principle of
 least surprise).
 
+@item -z
+@itemx --compress
+Compress existing POSIX tar archives aligning the lzip members to the tar
+members with choice of granularity (---bsolid by default, ---dsolid works
+like ---asolid). The input archives are kept unchanged. Existing compressed
+archives are not overwritten. A hyphen @samp{-} used as the name of an input
+archive reads from standard input and writes to standard output (unless the
+option @samp{--output} is used). Tarlz can be used as compressor for GNU tar
+using a command like @w{@samp{tar -c -Hustar foo | tarlz -z -o foo.tar.lz}}.
+Note that tarlz only works reliably on archives without global headers, or
+with global headers whose content can be ignored.
+
+The compression is reversible, including any garbage present after the EOF
+blocks. Tarlz stops parsing after the first EOF block is found, and then
+compresses the rest of the archive. Unless solid compression is requested,
+the EOF blocks are compressed in a lzip member separated from the preceding
+members and from any non-zero garbage following the EOF blocks.
+@samp{--compress} implies plzip argument style, not tar style. Each input
+archive is compressed to a file with the extension @samp{.lz} added unless
+the option @samp{--output} is used. When @samp{--output} is used, only one
+input archive can be specified. @samp{-f} can't be used with
+@samp{--compress}.
+
 @item -0 .. -9
-Set the compression level for @samp{--create} and @samp{--append}. The
+Set the compression level for @samp{--create}, @samp{--append}, and
-default compression level is @samp{-6}. Like lzip, tarlz also minimizes the
+@samp{--compress}. The default compression level is @samp{-6}. Like lzip,
-dictionary size of the lzip members it creates, reducing the amount of
+tarlz also minimizes the dictionary size of the lzip members it creates,
-memory required for decompression.
+reducing the amount of memory required for decompression.
 
 @multitable {Level} {Dictionary size} {Match length limit}
 @item Level @tab Dictionary size @tab Match length limit
@@ -446,6 +477,14 @@ the value of @samp{LZ_API_VERSION} (if defined).
 @xref{Library version,,,lzlib}.
 @end ifnothtml
 
+@item --warn-newer
+During archive creation, warn if any file being archived has a modification
+time newer than the archive creation time. This option may slow archive
+creation somewhat because it makes an extra call to @samp{stat} after
+archiving each file, but it guarantees that file contents were not modified
+during the creation of the archive. Note that the file must be at least one
+second newer than the archive for it to be detected as newer.
+
 @ignore
 @item --permissive
 Allow some violations of the archive format, like consecutive extended
@@ -457,7 +496,7 @@ keyword appearing in the same block of extended records.
 
 Exit status: 0 for a normal exit, 1 for environmental problems (file not
 found, files differ, invalid flags, I/O errors, etc), 2 to indicate a
-corrupt or invalid input file, 3 for an internal consistency error (eg, bug)
+corrupt or invalid input file, 3 for an internal consistency error (e.g. bug)
 which caused tarlz to panic.
 
 
@@ -477,7 +516,13 @@ The last three characters are the period, underscore, and hyphen-minus
 characters, respectively.
 
 File names are identifiers. Therefore, archiving works better when file
-names use only the portable character set without spaces added.
+names use only the portable character set without spaces added. Unicode is
+for human consumption. It should be
+@uref{http://www.gnu.org/software/moe/manual/moe_manual.html#why-not-Unicode,,avoided}
+in computing environments, specially in file names.
+@ifnothtml
+@xref{why not Unicode,,,moe}.
+@end ifnothtml
 
 
 @node File format
@@ -796,8 +841,9 @@ integrity checking of lzip may not be able to detect the corruption before
 the metadata has been used, for example, to create a new file in the wrong
 place.
 
-Because of the above, tarlz protects the extended records with a CRC in a
+Because of the above, tarlz protects the extended records with a Cyclic
-way compatible with standard tar tools. @xref{key_crc32}.
+Redundancy Check (CRC) in a way compatible with standard tar tools.
+@xref{key_crc32}.
 
 @sp 1
 @anchor{flawed-compat}
@@ -818,7 +864,9 @@ To avoid this problem, tarlz writes extended headers with all fields zeroed
 except size, chksum, typeflag, magic and version. This prevents old tar
 programs from extracting the extended records as a file in the wrong place.
 Tarlz also sets to zero those fields of the ustar header overridden by
-extended records.
+extended records. Finally, tarlz skips members without name when decoding
+except when listing. This is needed to detect certain format violations
+during parallel extraction.
 
 If an extended header is required for any reason (for example a file size
 larger than @w{8 GiB} or a link name longer than 100 bytes), tarlz moves the
@@ -940,11 +988,20 @@ error be avoided.
 @chapter Limitations of parallel tar decoding
 @cindex parallel tar decoding
 
-Safely decoding an arbitrary tar archive in parallel is impossible. For
+Safely decoding an arbitrary tar archive in parallel is only possible if one
-example, if a tar archive containing another tar archive is decoded starting
+decodes the headers sequentially first. For example, if a tar archive
-from some position other than the beginning, there is no way to know if the
+containing another tar archive is decoded starting from some position other
-first header found there belongs to the outer tar archive or to the inner
+than the beginning, there is no way to know if the first header found there
-tar archive. Tar is a format inherently serial; it was designed for tapes.
+belongs to the outer tar archive or to the inner tar archive. Tar is a
+format inherently serial; it was designed for tapes.
+
+The pax format is even more serial than the ustar format. Two headers need
+to be decoded sequentially for each file. The extended header may even need
+parsing to reveal something as basic as file size. If a thread decodes the
+ustar header skipping the preceding extended header, it may extract a file
+of incorrect size at the wrong place. Moreover, a pax archive with global
+headers can't be decoded in parallel because each thread can't know about
+the global headers decoded by other threads.
 
 In the case of compressed tar archives, the start of each compressed block
 determines one point through which the tar archive can be decoded in
@@ -1131,6 +1188,17 @@ directory @samp{destdir}.
 tarlz -C sourcedir -c . | tarlz -C destdir -x
 @end example
 
+@sp 1
+@noindent
+Example 9: Compress the existing POSIX archive @samp{archive.tar} and write
+the output to @samp{archive.tar.lz}. Compress each member individually for
+maximum availability. (If one member in the compressed archive gets damaged,
+the other members can still be extracted).
+
+@example
+tarlz -z --no-solid archive.tar
+@end example
+
 
 @node Problems
 @chapter Reporting bugs

exclude.cc

@@ -17,14 +17,8 @@
 
 #define _FILE_OFFSET_BITS 64
 
-#include <climits>
 #include <cstdlib>
-#include <cstring>
-#include <string>
-#include <vector>
 #include <fnmatch.h>
-#include <pthread.h>	// for tarlz.h
-#include <stdint.h>
 
 #include "tarlz.h"
 

extended.cc

@@ -18,14 +18,8 @@
 #define _FILE_OFFSET_BITS 64
 
 #include <cctype>
-#include <climits>
 #include <cstdio>
 #include <cstdlib>
-#include <cstring>
-#include <string>
-#include <vector>
-#include <pthread.h>	// for tarlz.h
-#include <stdint.h>
 
 #include "tarlz.h"
 
@@ -173,7 +167,7 @@ long long Extended::format_block( Resizable_buffer & rbuf ) const
   if( edsize_ <= 0 ) return 0;			// no extended data
   if( edsize_ >= 1LL << 33 ) return -1;		// too much extended data
   if( !rbuf.resize( bufsize ) ) return -1;	// extended block buffer
-  uint8_t * const header = (uint8_t *)rbuf();	// extended header
+  uint8_t * const header = rbuf.u8();		// extended header
   char * const buf = rbuf() + header_size;	// extended records
   init_tar_header( header );
   header[typeflag_o] = tf_extended;		// fill only required fields

lzip_index.cc

@@ -19,12 +19,10 @@
 
 #include <algorithm>
 #include <cerrno>
-#include <climits>	// for tarlz.h
 #include <cstdio>
 #include <cstring>
 #include <string>
 #include <vector>
-#include <pthread.h>	// for tarlz.h
 #include <stdint.h>
 #include <unistd.h>
 

main.cc

@@ -18,34 +18,30 @@
    Exit status: 0 for a normal exit, 1 for environmental problems (file not
    found, files differ, invalid flags, I/O errors, etc), 2 to indicate a
    corrupt or invalid input file, 3 for an internal consistency error
-   (eg, bug) which caused tarlz to panic.
+   (e.g. bug) which caused tarlz to panic.
 */
 
 #define _FILE_OFFSET_BITS 64
 
 #include <cctype>
 #include <cerrno>
-#include <climits>
 #include <cstdio>
 #include <cstdlib>
-#include <cstring>
 #include <ctime>
-#include <string>
-#include <vector>
 #include <fcntl.h>
 #include <pthread.h>
-#include <stdint.h>
+#include <stdint.h>	// for lzlib.h
 #include <unistd.h>
 #include <sys/stat.h>
 #include <grp.h>
 #include <pwd.h>
 #include <lzlib.h>
-#if defined(__OS2__)
+#if defined __OS2__
 #include <io.h>
 #endif
 
-#include "arg_parser.h"
 #include "tarlz.h"
+#include "arg_parser.h"
 
 #ifndef O_BINARY
 #define O_BINARY 0
@@ -56,10 +52,10 @@
 #endif
 
 int verbosity = 0;
+const char * const program_name = "tarlz";
 
 namespace {
 
-const char * const program_name = "tarlz";
 const char * const program_year = "2021";
 const char * invocation_name = program_name;		// default value
 
@@ -101,12 +97,14 @@ void show_help( const long num_online )
                "  -h, --dereference           follow symlinks; archive the files they point to\n"
                "      --mtime=<date>          use <date> as mtime for files added to archive\n"
                "  -n, --threads=<n>           set number of (de)compression threads [%ld]\n"
+               "  -o, --output=<file>         compress to <file>\n"
                "  -p, --preserve-permissions  don't subtract the umask on extraction\n"
                "  -q, --quiet                 suppress all messages\n"
                "  -r, --append                append files to the end of an archive\n"
                "  -t, --list                  list the contents of an archive\n"
                "  -v, --verbose               verbosely list files processed\n"
                "  -x, --extract               extract files/directories from an archive\n"
+               "  -z, --compress              compress existing POSIX tar archives\n"
                "  -0 .. -9                    set compression level [default 6]\n"
                "      --uncompressed          don't compress the archive created\n"
                "      --asolid                create solidly compressed appendable archive\n"
@@ -121,6 +119,7 @@ void show_help( const long num_online )
                "      --missing-crc           exit with error status if missing extended CRC\n"
                "      --out-slots=<n>         number of 1 MiB output packets buffered [64]\n"
                "      --check-lib             compare version of lzlib.h with liblz.{a,so}\n"
+               "      --warn-newer            warn if any file is newer than the archive\n"
 /*              "      --permissive            allow repeated extended headers and records\n"*/,
                num_online );
   if( verbosity >= 1 )
@@ -129,7 +128,7 @@ void show_help( const long num_online )
     }
   std::printf( "\nExit status: 0 for a normal exit, 1 for environmental problems (file not\n"
                "found, files differ, invalid flags, I/O errors, etc), 2 to indicate a\n"
-               "corrupt or invalid input file, 3 for an internal consistency error (eg, bug)\n"
+               "corrupt or invalid input file, 3 for an internal consistency error (e.g. bug)\n"
                "which caused tarlz to panic.\n"
                "\nReport bugs to lzip-bug@nongnu.org\n"
                "Tarlz home page: http://www.nongnu.org/lzip/tarlz.html\n" );
@@ -315,15 +314,17 @@ int open_instream( const std::string & name )
 
 
 int open_outstream( const std::string & name, const bool create,
-                    Resizable_buffer * const rbufp )
+                    Resizable_buffer * const rbufp, const bool force )
   {
-  const int flags = (create ? O_CREAT | O_WRONLY | O_TRUNC : O_RDWR) | O_BINARY;
+  const int cflags = O_CREAT | O_WRONLY | ( force ? O_TRUNC : O_EXCL );
+  const int flags = ( create ? cflags : O_RDWR ) | O_BINARY;
   const mode_t outfd_mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
 
   const int outfd = open( name.c_str(), flags, outfd_mode );
   if( outfd < 0 )
     {
-    const char * msg = create ? "Can't create file" : "Error opening file";
+    const char * msg = !create ? "Error opening file" :
+      ( ( errno == EEXIST ) ? "Skipping file" : "Can't create file" );
     if( !rbufp ) show_file_error( name.c_str(), msg, errno );
     else
       snprintf( (*rbufp)(), (*rbufp).size(), "%s: %s: %s\n", name.c_str(),
@@ -334,10 +335,10 @@ int open_outstream( const std::string & name, const bool create,
 
 
 /* This can be called from any thread, main thread or sub-threads alike,
-   since they all call common helper functions that call cleanup_and_fail()
+   since they all call common helper functions that call exit_fail_mt()
    in case of an error.
 */
-void cleanup_and_fail( const int retval )
+void exit_fail_mt( const int retval )
   {
   // calling 'exit' more than once results in undefined behavior
   static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
@@ -395,7 +396,7 @@ int main( const int argc, const char * const argv[] )
 
   enum { opt_ano = 256, opt_aso, opt_bso, opt_chk, opt_crc, opt_dbg, opt_del,
          opt_dso, opt_exc, opt_grp, opt_hlp, opt_id, opt_kd, opt_mti, opt_nso,
-         opt_out, opt_own, opt_per, opt_sol, opt_un };
+         opt_out, opt_own, opt_per, opt_sol, opt_un, opt_wn };
   const Arg_parser::Option options[] =
     {
     { '0', 0,                      Arg_parser::no  },
@@ -417,6 +418,7 @@ int main( const int argc, const char * const argv[] )
     { 'h', "dereference",          Arg_parser::no  },
     { 'H', "format",               Arg_parser::yes },
     { 'n', "threads",              Arg_parser::yes },
+    { 'o', "output",               Arg_parser::yes },
     { 'p', "preserve-permissions", Arg_parser::no  },
     { 'q', "quiet",                Arg_parser::no  },
     { 'r', "append",               Arg_parser::no  },
@@ -424,6 +426,7 @@ int main( const int argc, const char * const argv[] )
     { 'v', "verbose",              Arg_parser::no  },
     { 'V', "version",              Arg_parser::no  },
     { 'x', "extract",              Arg_parser::no  },
+    { 'z', "compress",             Arg_parser::no  },
     { opt_ano, "anonymous",        Arg_parser::no  },
     { opt_aso, "asolid",           Arg_parser::no  },
     { opt_bso, "bsolid",           Arg_parser::no  },
@@ -444,6 +447,7 @@ int main( const int argc, const char * const argv[] )
     { opt_per, "permissive",       Arg_parser::no  },
     { opt_sol, "solid",            Arg_parser::no  },
     { opt_un,  "uncompressed",     Arg_parser::no  },
+    { opt_wn,  "warn-newer",       Arg_parser::no  },
     {  0, 0,                       Arg_parser::no  } };
 
   const Arg_parser parser( argc, argv, options, true );	// in_order
@@ -463,7 +467,8 @@ int main( const int argc, const char * const argv[] )
       {
       if( parser.argument( argind ).empty() )
         { show_error( "Empty non-option argument." ); return 1; }
-      ++cl_opts.filenames; continue;
+      if( parser.argument( argind ) != "-" ) cl_opts.filenames_given = true;
+      ++cl_opts.num_files; continue;
       }
     const std::string & sarg = parser.argument( argind );
     const char * const arg = sarg.c_str();
@@ -482,6 +487,7 @@ int main( const int argc, const char * const argv[] )
       case 'h': cl_opts.dereference = true; break;
       case 'H': break;					// ignore format
       case 'n': cl_opts.num_workers = getnum( arg, 0, max_workers ); break;
+      case 'o': cl_opts.output_filename = sarg; break;
       case 'p': cl_opts.preserve_permissions = true; break;
       case 'q': verbosity = -1; break;
       case 'r': set_mode( cl_opts.program_mode, m_append ); break;
@@ -489,6 +495,7 @@ int main( const int argc, const char * const argv[] )
       case 'v': if( verbosity < 4 ) ++verbosity; break;
       case 'V': show_version(); return 0;
       case 'x': set_mode( cl_opts.program_mode, m_extract ); break;
+      case 'z': set_mode( cl_opts.program_mode, m_compress ); break;
       case opt_ano: set_owner( cl_opts.owner, "root" );
                     set_group( cl_opts.group, "root" ); break;
       case opt_aso: cl_opts.solidity = asolid; break;
@@ -510,15 +517,19 @@ int main( const int argc, const char * const argv[] )
       case opt_per: cl_opts.permissive = true; break;
       case opt_sol: cl_opts.solidity = solid; break;
       case opt_un:  cl_opts.level = -1; break;
+      case opt_wn:  cl_opts.warn_newer = true; break;
       default : internal_error( "uncaught option" );
       }
     } // end process options
 
-#if !defined LZ_API_VERSION || LZ_API_VERSION < 1	// compile-time test
+#if !defined LZ_API_VERSION || LZ_API_VERSION < 1012	// compile-time test
-#error "lzlib 1.8 or newer needed."
+#error "lzlib 1.12 or newer needed."
 #endif
+  if( LZ_api_version() < 1012 )				// runtime test
+    { show_error( "Wrong library version. At least lzlib 1.12 is required." );
+      return 1; }
 
-#if defined(__MSVCRT__) || defined(__OS2__)
+#if defined __OS2__
   setmode( STDIN_FILENO, O_BINARY );
   setmode( STDOUT_FILENO, O_BINARY );
 #endif
@@ -531,6 +542,7 @@ int main( const int argc, const char * const argv[] )
     case m_none:   show_error( "Missing operation.", 0, true ); return 1;
     case m_append:
     case m_create:      return encode( cl_opts );
+    case m_compress:    return compress( cl_opts );
     case m_concatenate: return concatenate( cl_opts );
     case m_delete:      return delete_members( cl_opts );
     case m_diff:

tarlz.h

@@ -15,6 +15,11 @@
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+#include <climits>
+#include <cstring>
+#include <string>
+#include <vector>
+#include <stdint.h>
 #include <sys/types.h>
 
 #define max_file_size ( LLONG_MAX - header_size )
@@ -41,7 +46,7 @@ enum Typeflag {
 const uint8_t ustar_magic[magic_l] =
   { 0x75, 0x73, 0x74, 0x61, 0x72, 0 };			// "ustar\0"
 
-inline bool verify_ustar_magic( const uint8_t * const header )
+inline bool verify_ustar_magic( const Tar_header header )
   { return std::memcmp( header + magic_o, ustar_magic, magic_l ) == 0; }
 
 inline void init_tar_header( Tar_header header )    // set magic and version
@@ -106,7 +111,10 @@ public:
       }
     return true;
     }
-  char * operator()() const { return p; }
+  char * operator()() { return p; }
+  const char * operator()() const { return p; }
+  uint8_t * u8() { return (uint8_t *)p; }
+  const uint8_t * u8() const { return (const uint8_t *)p; }
   unsigned long size() const { return size_; }
   };
 
@@ -222,6 +230,25 @@ public:
   };
 
 
+struct Lzma_options
+  {
+  int dictionary_size;		// 4 KiB .. 512 MiB
+  int match_len_limit;		// 5 .. 273
+  };
+const Lzma_options option_mapping[] =
+  {
+  {   65535,  16 },		// -0
+  { 1 << 20,   5 },		// -1
+  { 3 << 19,   6 },		// -2
+  { 1 << 21,   8 },		// -3
+  { 3 << 20,  12 },		// -4
+  { 1 << 22,  20 },		// -5
+  { 1 << 23,  36 },		// -6
+  { 1 << 24,  68 },		// -7
+  { 3 << 23, 132 },		// -8
+  { 1 << 25, 273 } };		// -9
+
+
 enum {
   min_dictionary_bits = 12,
   min_dictionary_size = 1 << min_dictionary_bits,
@@ -324,39 +351,44 @@ struct Lzip_trailer
   };
 
 
-enum Program_mode { m_none, m_append, m_concatenate, m_create, m_delete,
+enum Program_mode { m_none, m_append, m_compress, m_concatenate, m_create,
-                    m_diff, m_extract, m_list };
+                    m_delete, m_diff, m_extract, m_list };
 enum Solidity { no_solid, bsolid, dsolid, asolid, solid };
 class Arg_parser;
 struct Cl_options		// command line options
   {
   const Arg_parser & parser;
   std::string archive_name;
+  std::string output_filename;
   long long mtime;
   Program_mode program_mode;
   Solidity solidity;
   int data_size;
   int debug_level;
-  int filenames;
   int level;			// compression level, < 0 means uncompressed
+  int num_files;
   int num_workers;		// start this many worker threads
   int out_slots;
   int owner;
   int group;
   bool dereference;
+  bool filenames_given;
   bool ignore_ids;
   bool keep_damaged;
   bool missing_crc;
   bool permissive;
   bool preserve_permissions;
+  bool warn_newer;
 
   Cl_options( const Arg_parser & ap )
     : parser( ap ), mtime( -1 ), program_mode( m_none ), solidity( bsolid ),
-      data_size( 0 ), debug_level( 0 ), filenames( 0 ), level( 6 ),
+      data_size( 0 ), debug_level( 0 ), level( 6 ), num_files( 0 ),
       num_workers( -1 ), out_slots( 64 ), owner( -1 ), group( -1 ),
-      dereference( false ), ignore_ids( false ), keep_damaged( false ),
+      dereference( false ), filenames_given( false ), ignore_ids( false ),
-      missing_crc( false ), permissive( false ), preserve_permissions( false )
+      keep_damaged( false ), missing_crc( false ), permissive( false ),
-      {}
+      preserve_permissions( false ), warn_newer( false ) {}
+
+  bool to_stdout() const { return output_filename == "-"; }
   };
 
 
@@ -404,6 +436,9 @@ bool check_skip_filename( const Cl_options & cl_opts,
 mode_t get_umask();
 bool make_path( const std::string & name );
 
+// defined in compress.cc
+int compress( Cl_options & cl_opts );
+
 // defined in create.cc
 bool copy_file( const int infd, const int outfd, const long long max_size = -1 );
 bool writeblock_wrapper( const int outfd, const uint8_t * const buffer,
@@ -413,13 +448,14 @@ const char * remove_leading_dotslash( const char * const filename,
                                       const bool dotdot = false );
 bool fill_headers( const char * const filename, Extended & extended,
                    Tar_header header, long long & file_size, const int flag );
-bool block_is_full( const Extended & extended,
+bool block_is_full( const long long extended_size,
                     const unsigned long long file_size,
+                    const unsigned long long target_size,
                     unsigned long long & partial_data_size );
 void set_error_status( const int retval );
 int final_exit_status( int retval, const bool show_msg = true );
-unsigned ustar_chksum( const uint8_t * const header );
+unsigned ustar_chksum( const Tar_header header );
-bool verify_ustar_chksum( const uint8_t * const header );
+bool verify_ustar_chksum( const Tar_header header );
 bool has_lz_ext( const std::string & name );
 int concatenate( const Cl_options & cl_opts );
 int encode( Cl_options & cl_opts );
@@ -472,13 +508,14 @@ int seek_read( const int fd, uint8_t * const buf, const int size,
 
 // defined in main.cc
 extern int verbosity;
+extern const char * const program_name;
 struct stat;
 int hstat( const char * const filename, struct stat * const st,
            const bool dereference );
 int open_instream( const std::string & name );
 int open_outstream( const std::string & name, const bool create = true,
-                    Resizable_buffer * const rbufp = 0 );
+                    Resizable_buffer * const rbufp = 0, const bool force = true );
-void cleanup_and_fail( const int retval = 1 );	// terminate the program
+void exit_fail_mt( const int retval = 1 );	// terminate the program
 void show_error( const char * const msg, const int errcode = 0,
                  const bool help = false );
 void format_file_error( std::string & estr, const char * const filename,

testsuite/check.sh

@@ -57,20 +57,20 @@ bad6_lz="${testdir}"/test3_bad6.tar.lz
 eof="${testdir}"/eof.tar
 eof_lz="${testdir}"/eof.tar.lz
 fail=0
-lwarn=0
 lwarnc=0
 test_failed() { fail=1 ; printf " $1" ; [ -z "$2" ] || printf "($2)" ; }
-lzlib_1_11() { [ ${lwarn} = 0 ] &&
-	printf "\nwarning: testing --keep-damaged requires lzlib-1.11 or newer\n$1"
-	lwarn=1 ; }
 cyg_symlink() { [ ${lwarnc} = 0 ] &&
 	printf "\nwarning: your OS follows symbolic links to directories even when tarlz asks it not to\n$1"
 	lwarnc=1 ; }
 
 # Description of test files for tarlz:
 # test.txt.tar.lz:   1 member (test.txt).
-# t155.tar[.lz]:     directory + links + file + eof, all with 155 char names
+# t155.tar[.lz]:     directory + 3 links + file + eof, all with 155 char names
 # t155_fv?.tar[.lz]: like t155.tar but with 3 kinds of format violations
+# t155_fv1.tar[.lz]: extended header followed by EOF blocks
+# t155_fv2.tar[.lz]: extended header followed by global header
+# t155_fv3.tar[.lz]: consecutive extended headers
+# t155_fv[456].tar.lz: like t155_fv[123].tar.lz but violation starts member
 # tar_in_tlz1.tar.lz: 2 members (test.txt.tar test3.tar) 3 lzip members
 # tar_in_tlz2.tar.lz: 2 members (test.txt.tar test3.tar) 5 lzip members
 # ts_in_link.tar.lz: 4 symbolic links (link[1-4]) to / /dir/ dir/ dir(107/)
@@ -104,6 +104,7 @@ cyg_symlink() { [ ${lwarnc} = 0 ] &&
 # test3_gh?.tar.lz:  test3.tar.lz with global before bar split in 4 ways
 # test3_gh5.tar.lz:  test3.tar.lz with global in lzip member before foo
 # test3_gh6.tar.lz:  test3.tar.lz with global before foo in same member
+# test3_nn.tar[.lz]: test3.tar[.lz] with no name in bar member
 # test3_sm?.tar.lz:  test3.tar.lz with extended bar member split in 4 ways
 # tlz_in_tar1.tar:   1 member (test3.tar.lz) first magic damaged
 # tlz_in_tar2.tar:   2 members (foo test3.tar.lz) first magic damaged
@@ -549,12 +550,12 @@ printf "\ntesting --create..."
 
 # test --create
 cat "${in}" > test.txt || framework_failure
-"${TARLZ}" -0 -cf out.tar.lz test.txt || test_failed $LINENO
+"${TARLZ}" --warn-newer -0 -cf out.tar.lz test.txt || test_failed $LINENO
 rm -f test.txt || framework_failure
 "${TARLZ}" -xf out.tar.lz --missing-crc || test_failed $LINENO
 cmp "${in}" test.txt || test_failed $LINENO
 cat "${in}" > test.txt || framework_failure
-"${TARLZ}" --uncompressed -cf out.tar test.txt || test_failed $LINENO
+"${TARLZ}" --warn-newer --uncompressed -cf out.tar test.txt || test_failed $LINENO
 rm -f test.txt || framework_failure
 "${TARLZ}" -xf out.tar --missing-crc || test_failed $LINENO
 cmp "${in}" test.txt || test_failed $LINENO
@@ -691,9 +692,11 @@ else
       test_failed $LINENO $i
     rm -rf dir || framework_failure
   done
+  cmp out0 out2 || test_failed $LINENO
+  cmp out0 out6 || test_failed $LINENO
+  rm -f out0 out2 out6 || framework_failure
 fi
-cmp out0 out2 || test_failed $LINENO
+rm -f out.tar aout.tar foo bar baz || framework_failure
-rm -f out0 out2 out.tar aout.tar foo bar baz || framework_failure
 
 printf "\ntesting --delete..."
 
@@ -1035,6 +1038,62 @@ rm -f foo || framework_failure
 cmp cfoo foo || test_failed $LINENO
 rm -f foo || framework_failure
 
+printf "\ntesting --compress..."
+
+cat cfoo > foo || framework_failure
+cat cbar > bar || framework_failure
+cat cbaz > baz || framework_failure
+cat "${in}" > test.txt || framework_failure
+"${TARLZ}" --un -cf out.tar test.txt foo bar baz test.txt || test_failed $LINENO
+"${TARLZ}" --un -cf out3.tar foo bar baz || test_failed $LINENO
+cat out.tar > outz.tar || framework_failure
+cat out3.tar > out3z.tar || framework_failure
+#
+"${TARLZ}" -0 -z outz.tar out3z.tar || test_failed $LINENO
+"${TARLZ}" -q -tf outz.tar.lz || test_failed $LINENO
+"${TARLZ}" -q -tf out3z.tar.lz || test_failed $LINENO
+cat outz.tar.lz > out || test_failed $LINENO
+cat out3z.tar.lz > out3 || test_failed $LINENO
+rm -f out3z.tar.lz || framework_failure
+"${TARLZ}" -q -0 -z outz.tar out3z.tar
+[ $? = 1 ] || test_failed $LINENO
+cmp out outz.tar.lz || test_failed $LINENO
+cmp out3 out3z.tar.lz || test_failed $LINENO
+rm -f out out3 outz.tar.lz out3z.tar.lz || framework_failure
+#
+for i in --solid --no-solid ; do
+  "${TARLZ}" -0 -n0 $i -cf out.tar.lz test.txt foo bar baz test.txt || test_failed $LINENO $i
+  "${TARLZ}" -0 -z -o - $i out.tar | cmp out.tar.lz - || test_failed $LINENO $i
+  "${TARLZ}" -0 -n0 $i -cf out3.tar.lz foo bar baz || test_failed $LINENO $i
+  "${TARLZ}" -0 -z -o - $i out3.tar | cmp out3.tar.lz - || test_failed $LINENO $i
+  "${TARLZ}" -0 -z $i outz.tar out3z.tar || test_failed $LINENO $i
+  cmp out.tar.lz outz.tar.lz || test_failed $LINENO $i
+  cmp out3.tar.lz out3z.tar.lz || test_failed $LINENO $i
+  rm -f outz.tar.lz out3z.tar.lz || framework_failure
+done
+#
+"${TARLZ}" -0 -B8KiB -n0 --bsolid -cf out.tar.lz test.txt foo bar baz test.txt || test_failed $LINENO
+"${TARLZ}" -0 -B8KiB -z -o - --bsolid out.tar | cmp out.tar.lz - || test_failed $LINENO
+"${TARLZ}" -0 -B8KiB -z -o out --bsolid out.tar || test_failed $LINENO
+cmp out.tar.lz out || test_failed $LINENO
+"${TARLZ}" -0 -B8KiB -z --bsolid outz.tar || test_failed $LINENO
+cmp out.tar.lz outz.tar.lz || test_failed $LINENO
+rm -f out outz.tar.lz || framework_failure
+#
+"${TARLZ}" -0 -n0 --asolid -cf out.tar.lz test.txt foo bar baz test.txt || test_failed $LINENO
+"${TARLZ}" -0 -n0 --asolid -cf out3.tar.lz foo bar baz || test_failed $LINENO
+for i in --asolid --bsolid --dsolid ; do
+  cat out.tar | "${TARLZ}" -0 -z $i | cmp out.tar.lz - || test_failed $LINENO $i
+  "${TARLZ}" -0 -z -o out $i out.tar || test_failed $LINENO $i
+  cmp out.tar.lz out || test_failed $LINENO $i
+  "${TARLZ}" -0 -z $i outz.tar out3z.tar || test_failed $LINENO $i
+  cmp out.tar.lz outz.tar.lz || test_failed $LINENO $i
+  cmp out3.tar.lz out3z.tar.lz || test_failed $LINENO $i
+  rm -f out outz.tar.lz out3z.tar.lz || framework_failure
+done
+rm -f foo bar baz test.txt out.tar.lz out3.tar.lz out.tar outz.tar out3z.tar ||
+  framework_failure
+
 printf "\ntesting bad input..."
 
 # test --extract ".."
@@ -1104,15 +1163,30 @@ for i in 1 2 3 4 5 6 ; do
     test_failed $LINENO $i
   if [ "${ln_works}" = yes ] ; then
     mkdir dir2 || framework_failure
-		"${TARLZ}" -C dir2 -xf "${testdir}"/t155_fv${i}.tar.lz --permissive ||
+    "${TARLZ}" -n4 -C dir2 -xf "${testdir}"/t155_fv${i}.tar.lz --permissive ||
       test_failed $LINENO $i
     diff -ru dir1 dir2 || test_failed $LINENO $i
     rm -rf dir2 || framework_failure
   fi
 done
-if [ "${ln_works}" = yes ] ; then
+if [ "${ln_works}" = yes ] ; then rm -rf dir1 || framework_failure ; fi
-	rm -rf dir1 || framework_failure
+
-fi
+for i in "${testdir}"/test3_nn.tar "${testdir}"/test3_nn.tar.lz ; do
+  "${TARLZ}" -q -n0 -tf "$i" || test_failed $LINENO $i
+  "${TARLZ}" -q -n4 -tf "$i" || test_failed $LINENO $i
+  "${TARLZ}" -q -n0 -xf "$i" || test_failed $LINENO $i
+  "${TARLZ}" -n0 -df "$i" --ignore-ids || test_failed $LINENO $i
+  cmp cfoo foo || test_failed $LINENO $i
+  [ ! -e bar ] || test_failed $LINENO $i
+  cmp cbaz baz || test_failed $LINENO $i
+  rm -f foo bar baz || framework_failure
+  "${TARLZ}" -q -n4 -xf "$i" || test_failed $LINENO $i
+  "${TARLZ}" -n4 -df "$i" --ignore-ids || test_failed $LINENO $i
+  cmp cfoo foo || test_failed $LINENO $i
+  [ ! -e bar ] || test_failed $LINENO $i
+  cmp cbaz baz || test_failed $LINENO $i
+  rm -f foo bar baz || framework_failure
+done
 
 printf "\ntesting --keep-damaged..."
 
@@ -1126,7 +1200,7 @@ for i in "${inbad1}" "${inbad2}" ; do
 	"${TARLZ}" -q -n0 -xf "${i}.tar.lz" --keep-damaged
 	[ $? = 2 ] || test_failed $LINENO "$i"
 	[ -e test.txt ] || test_failed $LINENO "$i"
-	cmp "$i" test.txt 2> /dev/null || lzlib_1_11 "$LINENO $i"
+	cmp "$i" test.txt 2> /dev/null || test_failed $LINENO $i
 	rm -f test.txt || framework_failure
 done
 #
@@ -1152,7 +1226,7 @@ rm -f foo bar baz || framework_failure
 "${TARLZ}" -q -n0 -xf "${bad3_lz}" --keep-damaged
 [ $? = 2 ] || test_failed $LINENO
 cmp cfoo foo || test_failed $LINENO
-cmp cbar bar 2> /dev/null || lzlib_1_11 $LINENO
+cmp cbar bar 2> /dev/null || test_failed $LINENO
 cmp cbaz baz || test_failed $LINENO
 rm -f foo bar baz || framework_failure
 "${TARLZ}" -q -n0 -xf "${bad4_lz}"
@@ -1164,7 +1238,7 @@ rm -f foo bar baz || framework_failure
 "${TARLZ}" -q -n0 -xf "${bad4_lz}" --keep-damaged
 [ $? = 2 ] || test_failed $LINENO
 [ ! -e foo ] || test_failed $LINENO
-cmp cbar bar 2> /dev/null || lzlib_1_11 $LINENO
+cmp cbar bar 2> /dev/null || test_failed $LINENO
 cmp cbaz baz || test_failed $LINENO
 rm -f foo bar baz || framework_failure
 "${TARLZ}" -q -n0 -xf "${bad5_lz}"
@@ -1175,7 +1249,7 @@ cmp cbaz baz || test_failed $LINENO
 rm -f foo bar baz || framework_failure
 "${TARLZ}" -q -n0 -xf "${bad5_lz}" --keep-damaged
 [ $? = 2 ] || test_failed $LINENO
-cmp cfoo foo 2> /dev/null || lzlib_1_11 $LINENO
+cmp cfoo foo 2> /dev/null || test_failed $LINENO
 [ ! -e bar ] || test_failed $LINENO
 cmp cbaz baz || test_failed $LINENO
 rm -f foo bar baz || framework_failure