pdlzip/main.c

/* Pdlzip - LZMA lossless data compressor
   2009-08-14 : Igor Pavlov : Public domain
   Copyright (C) 2010-2024 Antonio Diaz Diaz.

   This program is free software. Redistribution and use in source and
   binary forms, with or without modification, are permitted provided
   that the following conditions are met:

   1. Redistributions of source code must retain the above copyright
   notice, this list of conditions, and the following disclaimer.

   2. Redistributions in binary form must reproduce the above copyright
   notice, this list of conditions, and the following disclaimer in the
   documentation and/or other materials provided with the distribution.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
*/
/*
   Exit status: 0 for a normal exit, 1 for environmental problems
   (file not found, invalid command-line options, I/O errors, etc), 2 to
   indicate a corrupt or invalid input file, 3 for an internal consistency
   error (e.g., bug) which caused pdlzip to panic.
*/

#define _FILE_OFFSET_BITS 64

#include <errno.h>
#include <fcntl.h>
#include <limits.h>		/* SSIZE_MAX */
#include <signal.h>
#include <stdbool.h>
#include <stdint.h>		/* SIZE_MAX */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <utime.h>
#include <sys/stat.h>
#if defined __MSVCRT__ || defined __OS2__ || defined __DJGPP__
#include <io.h>
#if defined __MSVCRT__
#define fchmod(x,y) 0
#define fchown(x,y,z) 0
#define SIGHUP SIGTERM
#define S_ISSOCK(x) 0
#ifndef S_IRGRP
#define S_IRGRP 0
#define S_IWGRP 0
#define S_IROTH 0
#define S_IWOTH 0
#endif
#endif
#if defined __DJGPP__
#define S_ISSOCK(x) 0
#define S_ISVTX 0
#endif
#endif

#include "carg_parser.h"
#include "lzip.h"
#include "LzmaDec.h"
#include "LzmaEnc.h"

#ifndef O_BINARY
#define O_BINARY 0
#endif

#if CHAR_BIT != 8
#error "Environments where CHAR_BIT != 8 are not supported."
#endif

#if ( defined  SIZE_MAX &&  SIZE_MAX < UINT_MAX ) || \
    ( defined SSIZE_MAX && SSIZE_MAX <  INT_MAX )
#error "Environments where 'size_t' is narrower than 'int' are not supported."
#endif

int verbosity = 0;
static void cleanup_and_fail( const int retval );
static void show_error( const char * const msg, const int errcode,
                        const bool help );
static void show_file_error( const char * const filename,
                             const char * const msg, const int errcode );
static void internal_error( const char * const msg );

static const char * const program_name = "pdlzip";
static const char * const program_year = "2024";
static const char * invocation_name = "pdlzip";		/* default value */

static const struct { const char * from; const char * to; } known_extensions[] = {
  { ".lz",   ""     },
  { ".tlz",  ".tar" },
  { ".lzma", ""     },
  { 0,       0      } };

struct Lzma_options
  {
  int dictionary_size;		/* 4 KiB .. 512 MiB */
  int match_len_limit;		/* 5 .. 273 */
  };

enum Mode { m_compress, m_decompress, m_test };

/* Variables used in signal handler context.
   They are not declared volatile because the handler never returns. */
static char * output_filename = 0;
static int outfd = -1;
static bool delete_output_on_interrupt = false;


static void show_help( void )
  {
  printf( "Pdlzip is a permissively licensed implementation of the lzip data\n"
          "compressor, intended for those who can't distribute (or even use) GPL\n"
          "licensed Free Software. The name of pdlzip comes from 'public domain lzip'.\n"
          "Pdlzip is written in C and is compatible with lzip 1.4 or newer.\n"
          "\nLzip is a lossless data compressor with a user interface similar to the one\n"
          "of gzip or bzip2. Lzip uses a simplified form of the 'Lempel-Ziv-Markov\n"
          "chain-Algorithm' (LZMA) stream format to maximize interoperability. The\n"
          "maximum dictionary size is 512 MiB so that any lzip file can be decompressed\n"
          "on 32-bit machines. Lzip provides accurate and robust 3-factor integrity\n"
          "checking. Lzip can compress about as fast as gzip (lzip -0) or compress most\n"
          "files more than bzip2 (lzip -9). Decompression speed is intermediate between\n"
          "gzip and bzip2. Lzip is better than gzip and bzip2 from a data recovery\n"
          "perspective. Lzip has been designed, written, and tested with great care to\n"
          "replace gzip and bzip2 as the standard general-purpose compressed format for\n"
          "Unix-like systems.\n"
          "\nPdlzip is also able to decompress legacy lzma-alone (.lzma) files.\n"
          "Lzma-alone is a very bad format; it is essentially a raw LZMA stream.\n"
          "If you keep any lzma-alone files, it is advisable to recompress them to\n"
          "lzip format. Lziprecover can convert some lzma-alone files to lzip format\n"
          "without recompressing.\n"
          "\nUsage: %s [options] [files]\n", invocation_name );
  printf( "\nOptions:\n"
          "  -h, --help                     display this help and exit\n"
          "  -V, --version                  output version information and exit\n"
          "  -a, --trailing-error           exit with error status if trailing data\n"
          "  -c, --stdout                   write to standard output, keep input files\n"
          "  -d, --decompress               decompress, test compressed file integrity\n"
          "  -f, --force                    overwrite existing output files\n"
          "  -F, --recompress               force re-compression of compressed files\n"
          "  -k, --keep                     keep (don't delete) input files\n"
          "  -m, --match-length=<bytes>     set match length limit in bytes [36]\n"
          "  -o, --output=<file>            write to <file>, keep input files\n"
          "  -q, --quiet                    suppress all messages\n"
          "  -s, --dictionary-size=<bytes>  set dictionary size limit in bytes [8 MiB]\n"
          "  -t, --test                     test compressed file integrity\n"
          "  -v, --verbose                  be verbose (a 2nd -v gives more)\n"
          "  -0 .. -9                       set compression level [default 6]\n"
          "      --fast                     alias for -0\n"
          "      --best                     alias for -9\n"
          "      --loose-trailing           allow trailing data seeming corrupt header\n"
          "\nIf no file names are given, or if a file is '-', pdlzip compresses or\n"
          "decompresses from standard input to standard output.\n"
          "Numbers may be followed by a multiplier: k = kB = 10^3 = 1000,\n"
          "Ki = KiB = 2^10 = 1024, M = 10^6, Mi = 2^20, G = 10^9, Gi = 2^30, etc...\n"
          "Dictionary sizes 12 to 27 are interpreted as powers of two, meaning 2^12 to\n"
          "2^27 bytes.\n"
          "\nThe bidimensional parameter space of LZMA can't be mapped to a linear scale\n"
          "optimal for all files. If your files are large, very repetitive, etc, you\n"
          "may need to use the options --dictionary-size and --match-length directly\n"
          "to achieve optimal performance. For example, -9m64 usually compresses\n"
          "executables more (and faster) than -9.\n"
          "\nTo extract all the files from archive 'foo.tar.lz', use the commands\n"
          "'tar -xf foo.tar.lz' or 'pdlzip -cd foo.tar.lz | tar -xf -'.\n"
          "\nExit status: 0 for a normal exit, 1 for environmental problems\n"
          "(file not found, invalid command-line options, I/O errors, etc), 2 to\n"
          "indicate a corrupt or invalid input file, 3 for an internal consistency\n"
          "error (e.g., bug) which caused pdlzip to panic.\n"
          "\nPdlzip includes public domain compression/decompression code from the LZMA\n"
          "SDK (Software Development Kit) written by Igor Pavlov.\n"
          "\nReport bugs to lzip-bug@nongnu.org\n"
          "Pdlzip home page: http://www.nongnu.org/lzip/pdlzip.html\n" );
  }


static void show_version( void )
  {
  printf( "%s %s\n", program_name, PROGVERSION );
  printf( "Copyright (C) %s Antonio Diaz Diaz.\n", program_year );
  printf( "Public Domain 2009 Igor Pavlov.\n"
          "License 2-clause BSD.\n"
          "This is free software: you are free to change and redistribute it.\n"
          "There is NO WARRANTY, to the extent permitted by law.\n" );
  }


/* assure at least a minimum size for buffer 'buf' */
static void * resize_buffer( void * buf, const unsigned min_size )
  {
  if( buf ) buf = realloc( buf, min_size );
  else buf = malloc( min_size );
  if( !buf ) { show_error( mem_msg, 0, false ); cleanup_and_fail( 1 ); }
  return buf;
  }


struct Pretty_print
  {
  const char * name;
  char * padded_name;
  const char * stdin_name;
  unsigned longest_name;
  bool first_post;
  };

static void Pp_init( struct Pretty_print * const pp,
                     const char * const filenames[], const int num_filenames )
  {
  pp->name = 0;
  pp->padded_name = 0;
  pp->stdin_name = "(stdin)";
  pp->longest_name = 0;
  pp->first_post = false;

  if( verbosity <= 0 ) return;
  const unsigned stdin_name_len = strlen( pp->stdin_name );
  int i;
  for( i = 0; i < num_filenames; ++i )
    {
    const char * const s = filenames[i];
    const unsigned len = (strcmp( s, "-" ) == 0) ? stdin_name_len : strlen( s );
    if( pp->longest_name < len ) pp->longest_name = len;
    }
  if( pp->longest_name == 0 ) pp->longest_name = stdin_name_len;
  }

static void Pp_set_name( struct Pretty_print * const pp,
                         const char * const filename )
  {
  unsigned name_len, padded_name_len, i = 0;

  if( filename && filename[0] && strcmp( filename, "-" ) != 0 )
    pp->name = filename;
  else pp->name = pp->stdin_name;
  name_len = strlen( pp->name );
  padded_name_len = max( name_len, pp->longest_name ) + 4;
  pp->padded_name = resize_buffer( pp->padded_name, padded_name_len + 1 );
  while( i < 2 ) pp->padded_name[i++] = ' ';
  while( i < name_len + 2 ) { pp->padded_name[i] = pp->name[i-2]; ++i; }
  pp->padded_name[i++] = ':';
  while( i < padded_name_len ) pp->padded_name[i++] = ' ';
  pp->padded_name[i] = 0;
  pp->first_post = true;
  }

static void Pp_reset( struct Pretty_print * const pp )
  { if( pp->name && pp->name[0] ) pp->first_post = true; }

static void Pp_show_msg( struct Pretty_print * const pp, const char * const msg )
  {
  if( verbosity < 0 ) return;
  if( pp->first_post )
    {
    pp->first_post = false;
    fputs( pp->padded_name, stderr );
    if( !msg ) fflush( stderr );
    }
  if( msg ) fprintf( stderr, "%s\n", msg );
  }


static void show_header( const unsigned dictionary_size )
  {
  enum { factor = 1024, n = 3 };
  const char * const prefix[n] = { "Ki", "Mi", "Gi" };
  const char * p = "";
  const char * np = "  ";
  unsigned num = dictionary_size;
  bool exact = ( num % factor == 0 );

  int i; for( i = 0; i < n && ( num > 9999 || ( exact && num >= factor ) ); ++i )
    { num /= factor; if( num % factor != 0 ) exact = false;
      p = prefix[i]; np = ""; }
  fprintf( stderr, "dict %s%4u %sB, ", np, num, p );
  }


/* separate numbers of 5 or more digits in groups of 3 digits using '_' */
static const char * format_num3( unsigned long long num )
  {
  enum { buffers = 8, bufsize = 4 * sizeof num, n = 10 };
  const char * const si_prefix = "kMGTPEZYRQ";
  const char * const binary_prefix = "KMGTPEZYRQ";
  static char buffer[buffers][bufsize];	/* circle of static buffers for printf */
  static int current = 0;
  int i;
  char * const buf = buffer[current++]; current %= buffers;
  char * p = buf + bufsize - 1;		/* fill the buffer backwards */
  *p = 0;	/* terminator */
  if( num > 1024 )
    {
    char prefix = 0;			/* try binary first, then si */
    for( i = 0; i < n && num != 0 && num % 1024 == 0; ++i )
      { num /= 1024; prefix = binary_prefix[i]; }
    if( prefix ) *(--p) = 'i';
    else
      for( i = 0; i < n && num != 0 && num % 1000 == 0; ++i )
        { num /= 1000; prefix = si_prefix[i]; }
    if( prefix ) *(--p) = prefix;
    }
  const bool split = num >= 10000;

  for( i = 0; ; )
    {
    *(--p) = num % 10 + '0'; num /= 10; if( num == 0 ) break;
    if( split && ++i >= 3 ) { i = 0; *(--p) = '_'; }
    }
  return p;
  }


void show_option_error( const char * const arg, const char * const msg,
                        const char * const option_name )
  {
  if( verbosity >= 0 )
    fprintf( stderr, "%s: '%s': %s option '%s'.\n",
             program_name, arg, msg, option_name );
  }


/* Recognized formats: <num>k, <num>Ki, <num>[MGTPEZYRQ][i] */
static unsigned long getnum( const char * const arg,
                             const char * const option_name,
                             const unsigned long llimit,
                             const unsigned long ulimit )
  {
  char * tail;
  errno = 0;
  unsigned long result = strtoul( arg, &tail, 0 );
  if( tail == arg )
    { show_option_error( arg, "Bad or missing numerical argument in",
                         option_name ); exit( 1 ); }

  if( !errno && tail[0] )
    {
    const unsigned factor = ( tail[1] == 'i' ) ? 1024 : 1000;
    int exponent = 0;				/* 0 = bad multiplier */
    int i;
    switch( tail[0] )
      {
      case 'Q': exponent = 10; break;
      case 'R': exponent = 9; break;
      case 'Y': exponent = 8; break;
      case 'Z': exponent = 7; break;
      case 'E': exponent = 6; break;
      case 'P': exponent = 5; break;
      case 'T': exponent = 4; break;
      case 'G': exponent = 3; break;
      case 'M': exponent = 2; break;
      case 'K': if( factor == 1024 ) exponent = 1; break;
      case 'k': if( factor == 1000 ) exponent = 1; break;
      }
    if( exponent <= 0 )
      { show_option_error( arg, "Bad multiplier in numerical argument of",
                           option_name ); exit( 1 ); }
    for( i = 0; i < exponent; ++i )
      {
      if( ulimit / factor >= result ) result *= factor;
      else { errno = ERANGE; break; }
      }
    }
  if( !errno && ( result < llimit || result > ulimit ) ) errno = ERANGE;
  if( errno )
    {
    if( verbosity >= 0 )
      fprintf( stderr, "%s: '%s': Value out of limits [%s,%s] in "
               "option '%s'.\n", program_name, arg, format_num3( llimit ),
               format_num3( ulimit ), option_name );
    exit( 1 );
    }
  return result;
  }


static int get_dict_size( const char * const arg, const char * const option_name )
  {
  char * tail;
  const long bits = strtol( arg, &tail, 0 );
  if( bits >= min_dictionary_bits &&
      bits <= max_dictionary_bits_c && *tail == 0 )
    return 1 << bits;
  return getnum( arg, option_name, min_dictionary_size, max_dictionary_size_c );
  }


static void set_mode( enum Mode * const program_modep, const enum Mode new_mode )
  {
  if( *program_modep != m_compress && *program_modep != new_mode )
    {
    show_error( "Only one operation can be specified.", 0, true );
    exit( 1 );
    }
  *program_modep = new_mode;
  }


static int extension_index( const char * const name )
  {
  int eindex;
  for( eindex = 0; known_extensions[eindex].from; ++eindex )
    {
    const char * const ext = known_extensions[eindex].from;
    const unsigned name_len = strlen( name );
    const unsigned ext_len = strlen( ext );
    if( name_len > ext_len &&
        strncmp( name + name_len - ext_len, ext, ext_len ) == 0 )
      return eindex;
    }
  return -1;
  }


static void set_c_outname( const char * const name, const bool filenames_given,
                           const bool force_ext )
  {
  /* zupdate < 1.9 depends on lzip adding the extension '.lz' to name when
     reading from standard input. */
  output_filename = resize_buffer( output_filename, strlen( name ) +
                                   strlen( known_extensions[0].from ) + 1 );
  strcpy( output_filename, name );
  if( force_ext ||
      ( !filenames_given && extension_index( output_filename ) < 0 ) )
    strcat( output_filename, known_extensions[0].from );
  }


static void set_d_outname( const char * const name, const int eindex )
  {
  const unsigned name_len = strlen( name );
  if( eindex >= 0 )
    {
    const char * const from = known_extensions[eindex].from;
    const unsigned from_len = strlen( from );
    if( name_len > from_len )
      {
      output_filename = resize_buffer( output_filename, name_len +
                                       strlen( known_extensions[eindex].to ) + 1 );
      strcpy( output_filename, name );
      strcpy( output_filename + name_len - from_len, known_extensions[eindex].to );
      return;
      }
    }
  output_filename = resize_buffer( output_filename, name_len + 4 + 1 );
  strcpy( output_filename, name );
  strcat( output_filename, ".out" );
  if( verbosity >= 1 )
    fprintf( stderr, "%s: %s: Can't guess original name -- using '%s'\n",
             program_name, name, output_filename );
  }


static int open_instream( const char * const name, struct stat * const in_statsp,
                          const enum Mode program_mode, const int eindex,
                          const bool one_to_one, const bool recompress )
  {
  if( program_mode == m_compress && !recompress && eindex >= 0 )
    {
    if( verbosity >= 0 )
      fprintf( stderr, "%s: %s: Input file already has '%s' suffix.\n",
               program_name, name, known_extensions[eindex].from );
    return -1;
    }
  int infd = open( name, O_RDONLY | O_BINARY );
  if( infd < 0 )
    show_file_error( name, "Can't open input file", errno );
  else
    {
    const int i = fstat( infd, in_statsp );
    const mode_t mode = in_statsp->st_mode;
    const bool can_read = ( i == 0 &&
                            ( S_ISBLK( mode ) || S_ISCHR( mode ) ||
                              S_ISFIFO( mode ) || S_ISSOCK( mode ) ) );
    if( i != 0 || ( !S_ISREG( mode ) && ( !can_read || one_to_one ) ) )
      {
      if( verbosity >= 0 )
        fprintf( stderr, "%s: %s: Input file is not a regular file%s.\n",
                 program_name, name, ( can_read && one_to_one ) ?
                 ",\n  and neither '-c' nor '-o' were specified" : "" );
      close( infd );
      infd = -1;
      }
    }
  return infd;
  }


static bool open_outstream( const bool force, const bool protect )
  {
  const mode_t usr_rw = S_IRUSR | S_IWUSR;
  const mode_t all_rw = usr_rw | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH;
  const mode_t outfd_mode = protect ? usr_rw : all_rw;
  int flags = O_CREAT | O_WRONLY | O_BINARY;
  if( force ) flags |= O_TRUNC; else flags |= O_EXCL;

  outfd = open( output_filename, flags, outfd_mode );
  if( outfd >= 0 ) delete_output_on_interrupt = true;
  else if( errno == EEXIST )
    show_file_error( output_filename,
                     "Output file already exists, skipping.", 0 );
  else
    show_file_error( output_filename, "Can't create output file", errno );
  return outfd >= 0;
  }


static void set_signals( void (*action)(int) )
  {
  signal( SIGHUP, action );
  signal( SIGINT, action );
  signal( SIGTERM, action );
  }


static void cleanup_and_fail( const int retval )
  {
  set_signals( SIG_IGN );			/* ignore signals */
  if( delete_output_on_interrupt )
    {
    delete_output_on_interrupt = false;
    show_file_error( output_filename, "Deleting output file, if it exists.", 0 );
    if( outfd >= 0 ) { close( outfd ); outfd = -1; }
    if( remove( output_filename ) != 0 && errno != ENOENT )
      show_error( "warning: deletion of output file failed", errno, false );
    }
  exit( retval );
  }


static void signal_handler( int sig )
  {
  if( sig ) {}				/* keep compiler happy */
  show_error( "Control-C or similar caught, quitting.", 0, false );
  cleanup_and_fail( 1 );
  }


static bool check_tty_in( const char * const input_filename, const int infd,
                          const enum Mode program_mode, int * const retval )
  {
  if( ( program_mode == m_decompress || program_mode == m_test ) &&
      isatty( infd ) )				/* for example /dev/tty */
    { show_file_error( input_filename,
                       "I won't read compressed data from a terminal.", 0 );
      close( infd ); set_retval( retval, 2 );
      if( program_mode != m_test ) cleanup_and_fail( *retval );
      return false; }
  return true;
  }

static bool check_tty_out( const enum Mode program_mode )
  {
  if( program_mode == m_compress && isatty( outfd ) )
    { show_file_error( output_filename[0] ?
                       output_filename : "(stdout)",
                       "I won't write compressed data to a terminal.", 0 );
      return false; }
  return true;
  }


/* Set permissions, owner, and times. */
static void close_and_set_permissions( const struct stat * const in_statsp )
  {
  bool warning = false;
  if( in_statsp )
    {
    const mode_t mode = in_statsp->st_mode;
    /* fchown in many cases returns with EPERM, which can be safely ignored. */
    if( fchown( outfd, in_statsp->st_uid, in_statsp->st_gid ) == 0 )
      { if( fchmod( outfd, mode ) != 0 ) warning = true; }
    else
      if( errno != EPERM ||
          fchmod( outfd, mode & ~( S_ISUID | S_ISGID | S_ISVTX ) ) != 0 )
        warning = true;
    }
  if( close( outfd ) != 0 )
    { show_file_error( output_filename, "Error closing output file", errno );
      cleanup_and_fail( 1 ); }
  outfd = -1;
  delete_output_on_interrupt = false;
  if( in_statsp )
    {
    struct utimbuf t;
    t.actime = in_statsp->st_atime;
    t.modtime = in_statsp->st_mtime;
    if( utime( output_filename, &t ) != 0 ) warning = true;
    }
  if( warning && verbosity >= 1 )
    show_file_error( output_filename,
                     "warning: can't change output file attributes", errno );
  }


static int compress( const int infd, const struct Lzma_options * const
                     encoder_options, struct Pretty_print * const pp )
  {
  int retval = 0;
  CLzmaEncHandle encoder = 0;
  Lzip_header header;
  Lh_set_magic( header );

  if( verbosity >= 1 ) Pp_show_msg( pp, 0 );
  if( Lh_set_dictionary_size( header, encoder_options->dictionary_size ) &&
      encoder_options->match_len_limit >= min_match_len_limit &&
      encoder_options->match_len_limit <= max_match_len )
    encoder = LzmaEnc_Init( Lh_get_dictionary_size( header ),
                            encoder_options->match_len_limit, infd, outfd );
  else internal_error( "invalid argument to encoder." );

  if( !encoder )
    {
    Pp_show_msg( pp, "Not enough memory. Try a smaller dictionary size." );
    return 1;
    }

  if( writeblock( outfd, header, Lh_size ) != Lh_size )
    { show_error( "Can't write output file", errno, false ); retval = 1; }
  else
    if( LzmaEnc_Encode( encoder ) != 0 )
      { Pp_show_msg( pp, "Encoder error." ); retval = 1; }
  LzmaEnc_Free( encoder );
  return retval;
  }


static void show_results( const long long data_size,
                          const long long member_size, const unsigned crc,
                          const unsigned dictionary_size, const bool lzip_mode )
  {
  if( verbosity >= 2 )
    {
    if( verbosity >= 4 ) show_header( dictionary_size );
    if( data_size == 0 || member_size == 0 )
      fputs( "no data compressed. ", stderr );
    else
      fprintf( stderr, "%6.3f:1, %5.2f%% ratio, %5.2f%% saved. ",
               (double)data_size / member_size,
               ( 100.0 * member_size ) / data_size,
               100.0 - ( ( 100.0 * member_size ) / data_size ) );
    if( verbosity >= 4 && lzip_mode ) fprintf( stderr, "CRC %08X, ", crc );
    if( verbosity >= 3 )
      fprintf( stderr, "%9llu out, %8llu in. ", data_size, member_size );
    if( !lzip_mode ) fputs( "lzma-alone, ", stderr );
    }
  }


#define IN_BUF_SIZE (1 << 16)
#define OUT_BUF_SIZE (1 << 16)

static bool read_inbuf( const int infd, uint8_t inBuf[],
                        int * const inPos, int * const inSize )
  {
  int rest;
  if( *inPos >= *inSize ) *inSize = 0;
  else if( *inPos > 0 )
    {
    memmove( inBuf, inBuf + *inPos, *inSize - *inPos );
    *inSize -= *inPos;
    }
  *inPos = 0;
  rest = IN_BUF_SIZE - *inSize;
  if( rest > 0 )
    {
    const int rd = readblock( infd, inBuf + *inSize, rest );
    if( rd < rest && errno )
      { show_error( "Read error", errno, false ); return false; }
    *inSize += rd;
    }
  return true;
  }


/* 5 bytes of LZMA properties and 8 bytes of uncompressed size */
enum { lzma_header_size = LZMA_PROPS_SIZE + 8 };

static int lzma_decode( uint64_t unpackSize, CLzmaDec *decoder, const int infd,
                        uint8_t inBuf[], int * const inPos,
                        int * const inSize, const unsigned dictionary_size,
                        const bool testing )
  {
  unsigned long long member_size = lzma_header_size, data_size = 0;
  uint8_t outBuf[OUT_BUF_SIZE];
  int outPos = 0;
  const bool thereIsSize = (unpackSize != (uint64_t)-1);

  for (;;)
    {
    uint32_t inProcessed;
    uint32_t outProcessed = OUT_BUF_SIZE - outPos;
    ELzmaFinishMode finishMode = LZMA_FINISH_ANY;
    ELzmaStatus status;

    if( *inPos >= *inSize && !read_inbuf( infd, inBuf, inPos, inSize ) )
      return 1;
    inProcessed = *inSize - *inPos;
    if (thereIsSize && outProcessed > unpackSize)
      {
      outProcessed = unpackSize;
      finishMode = LZMA_FINISH_END;
      }

    if( !LzmaDec_DecodeToBuf( decoder, outBuf + outPos, &outProcessed,
        inBuf + *inPos, &inProcessed, finishMode, &status ) )
      { show_error( "Data error.", 0, false ); return 2; }
    *inPos += inProcessed;
    member_size += inProcessed;
    outPos += outProcessed;
    unpackSize -= outProcessed;

    if( outfd >= 0 && writeblock( outfd, outBuf, outPos ) != outPos )
      { show_error( "Can't write output file", errno, false ); return 1; }

    data_size += outPos;
    outPos = 0;

    if( ( inProcessed == 0 && outProcessed == 0 ) ||
        ( thereIsSize && unpackSize == 0 ) )
      {
      if( ( thereIsSize && unpackSize != 0 ) ||
          ( !thereIsSize && status != LZMA_STATUS_FINISHED_WITH_MARK ) )
        { show_error( "Data error.", 0, false ); return 2; }
      show_results( data_size, member_size, 0, dictionary_size, false );
      if( verbosity >= 1 )
        fputs( testing ? "(apparently) ok\n" : "(apparently) done\n", stderr );
      return 0;
      }
    }
  }


static int lzip_decode( CLzmaDec *decoder, const int infd,
                        struct Pretty_print * const pp, uint8_t inBuf[],
                        int * const inPos, int * const inSize,
                        const unsigned dictionary_size )
  {
  unsigned long long member_size = Lh_size, data_size = 0;
  uint8_t outBuf[OUT_BUF_SIZE];
  int outPos = 0;
  uint32_t crc = 0xFFFFFFFFU;

  for (;;)
    {
    uint32_t inProcessed;
    uint32_t outProcessed = OUT_BUF_SIZE - outPos;
    ELzmaFinishMode finishMode = LZMA_FINISH_ANY;
    ELzmaStatus status;

    if( *inPos >= *inSize && !read_inbuf( infd, inBuf, inPos, inSize ) )
      return 1;
    if( *inPos >= *inSize )
      { Pp_show_msg( pp, "Unexpected EOF." ); return 2; }
    inProcessed = *inSize - *inPos;

    if( !LzmaDec_DecodeToBuf( decoder, outBuf + outPos, &outProcessed,
        inBuf + *inPos, &inProcessed, finishMode, &status ) )
      { Pp_show_msg( pp, "Data error." ); return 2; }
    *inPos += inProcessed;
    member_size += inProcessed;
    outPos += outProcessed;

    if( outfd >= 0 && writeblock( outfd, outBuf, outPos ) != outPos )
      { show_error( "Can't write output file", errno, false ); return 1; }

    CRC32_update_buf( &crc, outBuf, outPos );
    data_size += outPos;
    outPos = 0;

    if (inProcessed == 0 && outProcessed == 0)
      {
      Lzip_trailer trailer;
      int i;
      unsigned td_crc;
      unsigned long long td_size, tm_size;
      bool error = false;

      if( status != LZMA_STATUS_FINISHED_WITH_MARK )
        { Pp_show_msg( pp, "Data error." ); return 2; }
      if( *inSize - *inPos < Lt_size &&
          !read_inbuf( infd, inBuf, inPos, inSize ) ) return 1;
      if( *inSize - *inPos < Lt_size )
        {
        error = true;
        if( verbosity >= 0 )
          {
          Pp_show_msg( pp, 0 );
          fprintf( stderr, "Trailer truncated at trailer position %d;"
                           " some checks may fail.\n", *inSize - *inPos );
          }
        }
      for( i = 0; i < Lt_size && *inPos < *inSize; ++i )
        trailer[i] = inBuf[(*inPos)++];
      member_size += i;
      while( i < Lt_size ) trailer[i++] = 0;
      crc ^= 0xFFFFFFFFU;
      td_crc = Lt_get_data_crc( trailer );
      if( td_crc != crc )
        {
        error = true;
        if( verbosity >= 0 )
          {
          Pp_show_msg( pp, 0 );
          fprintf( stderr, "CRC mismatch; stored %08X, computed %08X\n",
                   td_crc, crc );
          }
        }
      td_size = Lt_get_data_size( trailer );
      if( td_size != data_size )
        {
        error = true;
        if( verbosity >= 0 )
          {
          Pp_show_msg( pp, 0 );
          fprintf( stderr, "Data size mismatch; stored %llu (0x%llX), computed %llu (0x%llX)\n",
                   td_size, td_size, data_size, data_size );
          }
        }
      tm_size = Lt_get_member_size( trailer );
      if( tm_size != member_size )
        {
        error = true;
        if( verbosity >= 0 )
          {
          Pp_show_msg( pp, 0 );
          fprintf( stderr, "Member size mismatch; stored %llu (0x%llX), computed %llu (0x%llX)\n",
                   tm_size, tm_size, member_size, member_size );
          }
        }
      if( error ) return 2;
      show_results( data_size, member_size, td_crc, dictionary_size, true );
      return 0;
      }
    }
  }


static int decompress( const int infd, struct Pretty_print * const pp,
                       const bool ignore_trailing, const bool loose_trailing,
                       const bool testing )
  {
  uint64_t unpackSize = 0;
  CLzmaDec decoder;
  uint8_t inBuf[IN_BUF_SIZE];
  int inPos = 0, inSize = 0;
  int retval = 0;
  bool lzip_mode = true;
  bool first_member;
  uint8_t raw_props[lzma_header_size];

  for( first_member = true; ; first_member = false )
    {
    int i;
    unsigned dictionary_size = 0;		/* keep gcc 3.3.6 quiet */
    Lzip_header header;
    if( inSize - inPos < lzma_header_size &&
        !read_inbuf( infd, inBuf, &inPos, &inSize ) ) return 1;
    const int size = inSize - inPos;
    for( i = 0; i < size && i < Lh_size; ++i )
      raw_props[i] = header[i] = inBuf[inPos++];
    if( size <= Lh_size )			/* End Of File */
      {
      if( first_member )
        { show_file_error( pp->name, "File ends unexpectedly at member header.", 0 );
          retval = 2; }
      else if( Lh_check_prefix( header, size ) )
        { Pp_show_msg( pp, "Truncated header in multimember file." );
          retval = 2; }
      else if( size > 0 && !ignore_trailing )
        { Pp_show_msg( pp, trailing_msg ); retval = 2; }
      break;
      }
    if( !Lh_check_magic( header ) )
      {
      if( !first_member )
        {
        if( !loose_trailing && Lh_check_corrupt( header ) )
          { Pp_show_msg( pp, "Corrupt header in multimember file." );
            retval = 2; }
        else if( !ignore_trailing )
          { Pp_show_msg( pp, trailing_msg ); retval = 2; }
        break;
        }
      if( inSize - inPos >= lzma_header_size - Lh_size )  /* try lzma-alone */
        {
        for( i = Lh_size; i < lzma_header_size; ++i )
          raw_props[i] = inBuf[inPos++];
        if( ( raw_props[12] == 0 || raw_props[12] == 0xFF ) &&
            raw_props[12] == raw_props[11] && raw_props[0] < (9 * 5 * 5) )
          {
          lzip_mode = false;
          dictionary_size = 0;
          for( i = 4; i >= 1; --i )
            { dictionary_size <<= 8; dictionary_size += raw_props[i]; }
          for( i = 7; i >= 0; --i )
            { unpackSize <<= 8; unpackSize += raw_props[LZMA_PROPS_SIZE+i]; }
          }
        }
      if( lzip_mode )
        {
        show_file_error( pp->name,
                         "Bad magic number (file not in lzip format).", 0 );
        retval = 2; break;
        }
      }
    if( lzip_mode )
      {
      if( !Lh_check_version( header ) )
        {
        if( verbosity >= 0 )
          { Pp_show_msg( pp, 0 );
            fprintf( stderr, "Version %d member format not supported.\n",
                     Lh_version( header ) ); }
        retval = 2; break;
        }
      dictionary_size = Lh_get_dictionary_size( header );
      if( !isvalid_ds( dictionary_size ) )
        { Pp_show_msg( pp, "Invalid dictionary size in member header." );
          retval = 2; break; }

      raw_props[0] = 93;		/* (45 * 2) + (9 * 0) + 3 */
      int ds = dictionary_size;
      for( i = 1; i <= 4; ++i ) { raw_props[i] = ds & 0xFF; ds >>= 8; }
      }

    if( verbosity >= 2 || ( verbosity == 1 && first_member ) )
      Pp_show_msg( pp, 0 );

    if( !LzmaDec_Init( &decoder, raw_props ) )
      { Pp_show_msg( pp, mem_msg ); return 1; }
    if( lzip_mode )
      retval = lzip_decode( &decoder, infd, pp, inBuf, &inPos, &inSize,
                            dictionary_size );
    else
      retval = lzma_decode( unpackSize, &decoder, infd, inBuf, &inPos,
                            &inSize, dictionary_size, testing );
    LzmaDec_Free(&decoder);
    if( retval != 0 || !lzip_mode ) break;
    if( verbosity >= 2 )
      { fputs( testing ? "ok\n" : "done\n", stderr ); Pp_reset( pp ); }
    }
  if( lzip_mode && verbosity == 1 && retval == 0 )
    fputs( testing ? "ok\n" : "done\n", stderr );
  return retval;
  }


CRC32 crc32;


/* Return the number of bytes really read.
   If (value returned < size) and (errno == 0), means EOF was reached.
*/
int readblock( const int fd, uint8_t * const buf, const int size )
  {
  int sz = 0;
  errno = 0;
  while( sz < size )
    {
    const int n = read( fd, buf + sz, size - sz );
    if( n > 0 ) sz += n;
    else if( n == 0 ) break;				/* EOF */
    else if( errno != EINTR ) break;
    errno = 0;
    }
  return sz;
  }


/* Return the number of bytes really written.
   If (value returned < size), it is always an error.
*/
int writeblock( const int fd, const uint8_t * const buf, const int size )
  {
  int sz = 0;
  errno = 0;
  while( sz < size )
    {
    const int n = write( fd, buf + sz, size - sz );
    if( n > 0 ) sz += n;
    else if( n < 0 && errno != EINTR ) break;
    errno = 0;
    }
  return sz;
  }


static void show_error( const char * const msg, const int errcode,
                        const bool help )
  {
  if( verbosity < 0 ) return;
  if( msg && msg[0] )
    fprintf( stderr, "%s: %s%s%s\n", program_name, msg,
             ( errcode > 0 ) ? ": " : "",
             ( errcode > 0 ) ? strerror( errcode ) : "" );
  if( help )
    fprintf( stderr, "Try '%s --help' for more information.\n",
             invocation_name );
  }


static void show_file_error( const char * const filename,
                             const char * const msg, const int errcode )
  {
  if( verbosity >= 0 )
    fprintf( stderr, "%s: %s: %s%s%s\n", program_name, filename, msg,
             ( errcode > 0 ) ? ": " : "",
             ( errcode > 0 ) ? strerror( errcode ) : "" );
  }


static void internal_error( const char * const msg )
  {
  if( verbosity >= 0 )
    fprintf( stderr, "%s: internal error: %s\n", program_name, msg );
  exit( 3 );
  }


int main( const int argc, const char * const argv[] )
  {
  /* Mapping from gzip/bzip2 style 0..9 compression levels to the
     corresponding LZMA compression parameters. */
  const struct Lzma_options option_mapping[] =
    {
    { 1 << 16,   5 },		/* -0 */
    { 1 << 20,   5 },		/* -1 */
    { 3 << 19,   6 },		/* -2 */
    { 1 << 21,   8 },		/* -3 */
    { 3 << 20,  12 },		/* -4 */
    { 1 << 22,  20 },		/* -5 */
    { 1 << 23,  36 },		/* -6 */
    { 1 << 24,  68 },		/* -7 */
    { 3 << 23, 132 },		/* -8 */
    { 1 << 25, 273 } };		/* -9 */
  struct Lzma_options encoder_options = option_mapping[6];  /* default = "-6" */
  const char * default_output_filename = "";
  enum Mode program_mode = m_compress;
  int i;
  bool force = false;
  bool ignore_trailing = true;
  bool keep_input_files = false;
  bool loose_trailing = false;
  bool recompress = false;
  bool to_stdout = false;
  if( argc > 0 ) invocation_name = argv[0];

  enum { opt_lt = 256 };
  const struct ap_Option options[] =
    {
    { '0', "fast",              ap_no  },
    { '1', 0,                   ap_no  },
    { '2', 0,                   ap_no  },
    { '3', 0,                   ap_no  },
    { '4', 0,                   ap_no  },
    { '5', 0,                   ap_no  },
    { '6', 0,                   ap_no  },
    { '7', 0,                   ap_no  },
    { '8', 0,                   ap_no  },
    { '9', "best",              ap_no  },
    { 'a', "trailing-error",    ap_no  },
    { 'b', "member-size",       ap_yes },
    { 'c', "stdout",            ap_no  },
    { 'd', "decompress",        ap_no  },
    { 'f', "force",             ap_no  },
    { 'F', "recompress",        ap_no  },
    { 'h', "help",              ap_no  },
    { 'k', "keep",              ap_no  },
    { 'm', "match-length",      ap_yes },
    { 'n', "threads",           ap_yes },
    { 'o', "output",            ap_yes },
    { 'q', "quiet",             ap_no  },
    { 's', "dictionary-size",   ap_yes },
    { 'S', "volume-size",       ap_yes },
    { 't', "test",              ap_no  },
    { 'v', "verbose",           ap_no  },
    { 'V', "version",           ap_no  },
    { opt_lt, "loose-trailing", ap_no  },
    {  0, 0,                    ap_no  } };

  CRC32_init();

  /* static because valgrind complains and memory management in C sucks */
  static struct Arg_parser parser;
  if( !ap_init( &parser, argc, argv, options, 0 ) )
    { show_error( mem_msg, 0, false ); return 1; }
  if( ap_error( &parser ) )				/* bad option */
    { show_error( ap_error( &parser ), 0, true ); return 1; }

  int argind = 0;
  for( ; argind < ap_arguments( &parser ); ++argind )
    {
    const int code = ap_code( &parser, argind );
    if( !code ) break;					/* no more options */
    const char * const pn = ap_parsed_name( &parser, argind );
    const char * const arg = ap_argument( &parser, argind );
    switch( code )
      {
      case '0': case '1': case '2': case '3': case '4':
      case '5': case '6': case '7': case '8': case '9':
                encoder_options = option_mapping[code-'0']; break;
      case 'a': ignore_trailing = false; break;
      case 'b': break;
      case 'c': to_stdout = true; break;
      case 'd': set_mode( &program_mode, m_decompress ); break;
      case 'f': force = true; break;
      case 'F': recompress = true; break;
      case 'h': show_help(); return 0;
      case 'k': keep_input_files = true; break;
      case 'm': encoder_options.match_len_limit =
                  getnum( arg, pn, min_match_len_limit, max_match_len ); break;
      case 'n': break;
      case 'o': if( strcmp( arg, "-" ) == 0 ) to_stdout = true;
                else { default_output_filename = arg; } break;
      case 'q': verbosity = -1; break;
      case 's': encoder_options.dictionary_size = get_dict_size( arg, pn );
                break;
      case 'S': break;
      case 't': set_mode( &program_mode, m_test ); break;
      case 'v': if( verbosity < 4 ) ++verbosity; break;
      case 'V': show_version(); return 0;
      case opt_lt: loose_trailing = true; break;
      default: internal_error( "uncaught option." );
      }
    } /* end process options */

#if defined __MSVCRT__ || defined __OS2__ || defined __DJGPP__
  setmode( STDIN_FILENO, O_BINARY );
  setmode( STDOUT_FILENO, O_BINARY );
#endif

  static const char ** filenames = 0;
  int num_filenames = max( 1, ap_arguments( &parser ) - argind );
  filenames = resize_buffer( filenames, num_filenames * sizeof filenames[0] );
  filenames[0] = "-";

  bool filenames_given = false;
  for( i = 0; argind + i < ap_arguments( &parser ); ++i )
    {
    filenames[i] = ap_argument( &parser, argind + i );
    if( strcmp( filenames[i], "-" ) != 0 ) filenames_given = true;
    }

  if( program_mode == m_test ) to_stdout = false;	/* apply overrides */
  if( program_mode == m_test || to_stdout ) default_output_filename = "";

  output_filename = resize_buffer( output_filename, 1 );
  output_filename[0] = 0;
  if( to_stdout && program_mode != m_test )	/* check tty only once */
    { outfd = STDOUT_FILENO; if( !check_tty_out( program_mode ) ) return 1; }
  else outfd = -1;

  const bool to_file = !to_stdout && program_mode != m_test &&
                       default_output_filename[0];
  if( !to_stdout && program_mode != m_test && ( filenames_given || to_file ) )
    set_signals( signal_handler );

  static struct Pretty_print pp;
  Pp_init( &pp, filenames, num_filenames );

  int failed_tests = 0;
  int retval = 0;
  const bool one_to_one = !to_stdout && program_mode != m_test && !to_file;
  bool stdin_used = false;
  struct stat in_stats;
  for( i = 0; i < num_filenames; ++i )
    {
    const char * input_filename = "";
    int infd;

    Pp_set_name( &pp, filenames[i] );
    if( strcmp( filenames[i], "-" ) == 0 )
      {
      if( stdin_used ) continue; else stdin_used = true;
      infd = STDIN_FILENO;
      if( !check_tty_in( pp.name, infd, program_mode, &retval ) ) continue;
      if( one_to_one ) { outfd = STDOUT_FILENO; output_filename[0] = 0; }
      }
    else
      {
      const int eindex = extension_index( input_filename = filenames[i] );
      infd = open_instream( input_filename, &in_stats, program_mode,
                            eindex, one_to_one, recompress );
      if( infd < 0 ) { set_retval( &retval, 1 ); continue; }
      if( !check_tty_in( pp.name, infd, program_mode, &retval ) ) continue;
      if( one_to_one )			/* open outfd after checking infd */
        {
        if( program_mode == m_compress )
          set_c_outname( input_filename, true, true );
        else set_d_outname( input_filename, eindex );
        if( !open_outstream( force, true ) )
          { close( infd ); set_retval( &retval, 1 ); continue; }
        }
      }

    if( one_to_one && !check_tty_out( program_mode ) )
      { set_retval( &retval, 1 ); return retval; }	/* don't delete a tty */

    if( to_file && outfd < 0 )		/* open outfd after checking infd */
      {
      if( program_mode == m_compress ) set_c_outname( default_output_filename,
                                       filenames_given, false );
      else
        { output_filename = resize_buffer( output_filename,
                            strlen( default_output_filename ) + 1 );
          strcpy( output_filename, default_output_filename ); }
      if( !open_outstream( force, false ) || !check_tty_out( program_mode ) )
        return 1;	/* check tty only once and don't try to delete a tty */
      }

    const struct stat * const in_statsp =
      ( input_filename[0] && one_to_one ) ? &in_stats : 0;
    int tmp;
    if( program_mode == m_compress )
      tmp = compress( infd, &encoder_options, &pp );
    else
      tmp = decompress( infd, &pp, ignore_trailing, loose_trailing,
                        program_mode == m_test );
    if( close( infd ) != 0 )
      { show_file_error( pp.name, "Error closing input file", errno );
        set_retval( &tmp, 1 ); }
    set_retval( &retval, tmp );
    if( tmp )
      { if( program_mode != m_test ) cleanup_and_fail( retval );
        else ++failed_tests; }

    if( delete_output_on_interrupt && one_to_one )
      close_and_set_permissions( in_statsp );
    if( input_filename[0] && !keep_input_files && one_to_one )
      remove( input_filename );
    }
  if( delete_output_on_interrupt )					/* -o */
    close_and_set_permissions( ( retval == 0 && !stdin_used &&
      filenames_given && num_filenames == 1 ) ? &in_stats : 0 );
  else if( outfd >= 0 && close( outfd ) != 0 )				/* -c */
    {
    show_error( "Error closing stdout", errno, false );
    set_retval( &retval, 1 );
    }
  if( failed_tests > 0 && verbosity >= 1 && num_filenames > 1 )
    fprintf( stderr, "%s: warning: %d %s failed the test.\n",
             program_name, failed_tests,
             ( failed_tests == 1 ) ? "file" : "files" );
  free( output_filename );
  free( filenames );
  ap_free( &parser );
  return retval;
  }