lziprecover/repair.cc

/*  Lziprecover - Data recovery tool for the lzip format
    Copyright (C) 2009-2016 Antonio Diaz Diaz.

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 2 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#define _FILE_OFFSET_BITS 64

#include <algorithm>
#include <cerrno>
#include <climits>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <string>
#include <vector>
#include <stdint.h>
#include <unistd.h>
#include <sys/stat.h>

#include "lzip.h"
#include "mtester.h"
#include "block.h"
#include "file_index.h"


namespace {

bool gross_damage( const long long msize, const uint8_t * const mbuffer )
  {
  enum { maxlen = 6 };		// max number of consecutive identical bytes
  long i = File_header::size;
  const long end = msize - File_trailer::size - maxlen;
  while( i < end )
    {
    const uint8_t byte = mbuffer[i];
    int len = 0;			// does not count the first byte
    while( mbuffer[++i] == byte ) if( ++len >= maxlen ) return true;
    }
  return false;
  }


int seek_write( const int fd, const uint8_t * const buf, const int size,
                const long long pos )
  {
  if( lseek( fd, pos, SEEK_SET ) == pos )
    return writeblock( fd, buf, size );
  return 0;
  }


// Return value: 0 = no change, 5 = repaired pos
int repair_dictionary_size( const long long msize, uint8_t * const mbuffer )
  {
  enum { dictionary_size_9 = 1 << 25 };		// dictionary size of option -9
  File_header & header = *(File_header *)mbuffer;
  unsigned dictionary_size = header.dictionary_size();
  File_trailer & trailer =
      *(File_trailer *)( mbuffer + msize - File_trailer::size );
  const unsigned long long data_size = trailer.data_size();
  const bool valid_ds = isvalid_ds( dictionary_size );
  if( valid_ds && dictionary_size >= data_size ) return 0;	// can't be bad

  if( !valid_ds || dictionary_size < dictionary_size_9 )
    {
    dictionary_size =
      std::min( data_size, (unsigned long long)dictionary_size_9 );
    if( dictionary_size < min_dictionary_size )
      dictionary_size = min_dictionary_size;
    LZ_mtester mtester( mbuffer, msize, dictionary_size );
    const int result = mtester.test_member();
    if( result == 0 )
      { header.dictionary_size( dictionary_size ); return 5; }	// fix DS
    if( result != 1 || mtester.max_distance() <= dictionary_size ||
        mtester.max_distance() > max_dictionary_size ) return 0;
    }
  if( data_size > dictionary_size_9 )
    {
    dictionary_size =
      std::min( data_size, (unsigned long long)max_dictionary_size );
    LZ_mtester mtester( mbuffer, msize, dictionary_size );
    if( mtester.test_member() == 0 )
      { header.dictionary_size( dictionary_size ); return 5; }	// fix DS
    }
  return 0;
  }


// Return value: -1 = master failed, 0 = begin reached, >0 = repaired pos
long repair_member( const long long mpos, const long long msize,
                    uint8_t * const mbuffer, const long begin, const long end,
                    const unsigned dictionary_size, const int verbosity )
  {
  for( long pos = end; pos >= begin && pos > end - 50000; )
    {
    const long min_pos = std::max( begin, pos - 100 );
    const unsigned long pos_limit = std::max( min_pos - 16, 0L );
    const LZ_mtester * master =
      prepare_master( mbuffer, msize, pos_limit, dictionary_size );
    if( !master ) return -1;
    for( ; pos >= min_pos; --pos )
      {
      if( verbosity >= 1 )
        {
        std::printf( "Trying position %llu \r", mpos + pos );
        std::fflush( stdout );
        }
      for( int j = 0; j < 255; ++j )
        {
        ++mbuffer[pos];
        if( test_member_rest( *master ) ) { delete master; return pos; }
        }
      ++mbuffer[pos];
      }
    delete master;
    }
  return 0;
  }

} // end namespace


int repair_file( const std::string & input_filename,
                 const std::string & default_output_filename,
                 const int verbosity, const bool force )
  {
  struct stat in_stats;
  const int infd = open_instream( input_filename.c_str(), &in_stats, true, true );
  if( infd < 0 ) return 1;

  Pretty_print pp( input_filename, verbosity );
  const File_index file_index( infd );
  if( file_index.retval() != 0 )
    { pp( file_index.error().c_str() ); return file_index.retval(); }

  output_filename = default_output_filename.empty() ?
                    insert_fixed( input_filename ) : default_output_filename;
  if( !force && file_exists( output_filename ) ) return 1;
  outfd = -1;
  for( long i = 0; i < file_index.members(); ++i )
    {
    const long long mpos = file_index.mblock( i ).pos();
    const long long msize = file_index.mblock( i ).size();
    if( !safe_seek( infd, mpos ) )
      cleanup_and_fail( 1 );
    long long failure_pos = 0;
    if( test_member_from_file( infd, msize, &failure_pos ) ) continue;
    if( failure_pos < File_header::size )		// End Of File
      { show_error( "Can't repair error in input file." );
        cleanup_and_fail( 2 ); }

    if( verbosity >= 1 )		// damaged member found
      {
      std::printf( "Repairing member %ld of %ld  (failure pos = %llu)\n",
                   i + 1, file_index.members(), mpos + failure_pos );
      std::fflush( stdout );
      }
    if( failure_pos >= msize - 8 ) failure_pos = msize - 8 - 1;
    uint8_t * const mbuffer = read_member( infd, mpos, msize );
    if( !mbuffer )
      cleanup_and_fail( 1 );
    const File_header & header = *(File_header *)mbuffer;
    const unsigned dictionary_size = header.dictionary_size();
    long pos = 0;
    if( !gross_damage( msize, mbuffer ) )
      {
      pos = repair_dictionary_size( msize, mbuffer );
      if( pos == 0 )
        pos = repair_member( mpos, msize, mbuffer, File_header::size + 1,
                             File_header::size + 5, dictionary_size, verbosity );
      if( pos == 0 )
        pos = repair_member( mpos, msize, mbuffer, File_header::size + 6,
                             failure_pos, dictionary_size, verbosity );
      }
    if( pos < 0 )
      cleanup_and_fail( 1 );
    if( pos > 0 )
      {
      if( outfd < 0 )		// first damaged member repaired
        {
        if( !safe_seek( infd, 0 ) ) return 1;
        if( !open_outstream( true, false ) ) { close( infd ); return 1; }
        if( !copy_file( infd, outfd ) )		// copy whole file
          cleanup_and_fail( 1 );
        }
      if( seek_write( outfd, mbuffer + pos, 1, mpos + pos ) != 1 )
        { show_error( "Error writing output file", errno );
          cleanup_and_fail( 1 ); }
      }
    delete[] mbuffer;
    if( verbosity >= 1 ) std::fputc( '\n', stdout );
    if( pos == 0 )
      {
      show_error( "Can't repair input file. Error is probably larger than 1 byte." );
      cleanup_and_fail( 2 );
      }
    }

  if( outfd < 0 )
    {
    if( verbosity >= 1 )
      std::fputs( "Input file has no errors. Recovery is not needed.\n", stdout );
    return 0;
    }
  if( close_outstream( &in_stats ) != 0 ) return 1;
  if( verbosity >= 1 )
    std::fputs( "Copy of input file repaired successfully.\n", stdout );
  return 0;
  }


int debug_delay( const std::string & input_filename, Block range,
                 const int verbosity )
  {
  struct stat in_stats;				// not used
  const int infd = open_instream( input_filename.c_str(), &in_stats, true, true );
  if( infd < 0 ) return 1;

  Pretty_print pp( input_filename, verbosity );
  const File_index file_index( infd );
  if( file_index.retval() != 0 )
    { pp( file_index.error().c_str() ); return file_index.retval(); }

  if( range.end() > file_index.file_end() )
    range.size( std::max( 0LL, file_index.file_end() - range.pos() ) );
  if( range.size() <= 0 )
    { if( verbosity >= 0 ) pp( "Nothing to do." ); return 0; }

  for( long i = 0; i < file_index.members(); ++i )
    {
    const Block & mb = file_index.mblock( i );
    if( !range.overlaps( mb ) ) continue;
    const long long mpos = file_index.mblock( i ).pos();
    const long long msize = file_index.mblock( i ).size();
    const unsigned dictionary_size = file_index.dictionary_size( i );
    if( verbosity >= 1 )
      {
      std::printf( "Finding max delay in member %ld of %ld  (mpos = %llu, msize = %llu)\n",
                   i + 1, file_index.members(), mpos, msize );
      std::fflush( stdout );
      }
    uint8_t * const mbuffer = read_member( infd, mpos, msize );
    if( !mbuffer ) return 1;
    long pos = std::max( range.pos() - mpos, File_header::size + 1LL );
    const long end = std::min( range.end() - mpos, msize );
    long max_delay = 0;
    while( pos < end )
      {
      const unsigned long pos_limit = std::max( pos - 16, 0L );
      const LZ_mtester * master =
        prepare_master( mbuffer, msize, pos_limit, dictionary_size );
      if( !master )
        { show_error( "Can't prepare master." ); return 1; }
      const long partial_end = std::min( pos + 100, end );
      for( ; pos < partial_end; ++pos )
        {
        if( verbosity >= 1 )
          {
          std::printf( "Delays in position %llu \r", mpos + pos );
          std::fflush( stdout );
          }
        int value = -1;
        for( int j = 0; j < 256; ++j )
          {
          ++mbuffer[pos];
          if( j == 255 ) break;
          long failure_pos = 0;
          if( test_member_rest( *master, &failure_pos ) ) continue;
          const long delay = failure_pos - pos;
          if( delay > max_delay ) { max_delay = delay; value = mbuffer[pos]; }
          }
        if( value >= 0 && verbosity >= 0 )
          {
          std::printf( "New max delay %lu at position %llu (0x%02X)\n",
                       max_delay, mpos + pos, value );
          std::fflush( stdout );
          }
        if( pos + max_delay >= msize ) { pos = end; break; }
        }
      delete master;
      }
    delete[] mbuffer;
    if( verbosity >= 1 ) std::fputc( '\n', stdout );
    }

  if( verbosity >= 1 ) std::fputs( "Done.\n", stdout );
  return 0;
  }


int debug_repair( const std::string & input_filename, const long long bad_pos,
                  const int verbosity, const uint8_t bad_value )
  {
  struct stat in_stats;				// not used
  const int infd = open_instream( input_filename.c_str(), &in_stats, true, true );
  if( infd < 0 ) return 1;

  Pretty_print pp( input_filename, verbosity );
  const File_index file_index( infd );
  if( file_index.retval() != 0 )
    { pp( file_index.error().c_str() ); return file_index.retval(); }

  long idx = 0;
  for( ; idx < file_index.members(); ++idx )
    if( file_index.mblock( idx ).includes( bad_pos ) ) break;
  if( idx >= file_index.members() )
    { if( verbosity >= 0 ) pp( "Nothing to do." ); return 0; }

  const long long mpos = file_index.mblock( idx ).pos();
  const long long msize = file_index.mblock( idx ).size();
  {
  long long failure_pos = 0;
  if( !safe_seek( infd, mpos ) ) return 1;
  if( !test_member_from_file( infd, msize, &failure_pos ) )
    {
    if( verbosity >= 0 )
      std::printf( "Member %ld of %ld already damaged  (failure pos = %llu)\n",
                   idx + 1, file_index.members(), mpos + failure_pos );
    return 1;
    }
  }
  uint8_t * const mbuffer = read_member( infd, mpos, msize );
  if( !mbuffer ) return 1;
  const File_header & header = *(File_header *)mbuffer;
  const unsigned dictionary_size = header.dictionary_size();
  const uint8_t good_value = mbuffer[bad_pos-mpos];
  mbuffer[bad_pos-mpos] = bad_value;
  long failure_pos = 0;
  if( bad_pos != 5 || isvalid_ds( header.dictionary_size() ) )
    {
    const LZ_mtester * master =
      prepare_master( mbuffer, msize, 0, header.dictionary_size() );
    if( !master )
      { show_error( "Can't prepare master." ); delete[] mbuffer; return 1; }
    if( test_member_rest( *master, &failure_pos ) )
      {
      if( verbosity >= 1 )
        std::fputs( "Member decompressed with no errors.\n", stdout );
      delete master;
      delete[] mbuffer;
      return 0;
      }
    delete master;
    }
  if( verbosity >= 1 )
    {
    std::printf( "Test repairing member %ld of %ld  (mpos = %llu, msize = %llu)\n"
                 "  (damage pos = %llu (0x%02X->0x%02X), failure pos = %llu)\n",
                 idx + 1, file_index.members(), mpos, msize,
                 bad_pos, good_value, bad_value, mpos + failure_pos );
    std::fflush( stdout );
    }
  if( failure_pos >= msize ) failure_pos = msize - 1;
  long pos = repair_dictionary_size( msize, mbuffer );
  if( pos == 0 )
    pos = repair_member( mpos, msize, mbuffer, File_header::size + 1,
                         File_header::size + 5, dictionary_size, verbosity );
  if( pos == 0 )
    pos = repair_member( mpos, msize, mbuffer, File_header::size + 6,
                         failure_pos, dictionary_size, verbosity );
  delete[] mbuffer;
  if( pos < 0 )
    { show_error( "Can't prepare master." ); return 1; }
  if( verbosity >= 1 ) std::fputc( '\n', stdout );
  if( pos == 0 ) internal_error( "can't repair input file." );
  if( verbosity >= 1 )
    std::fputs( "Member repaired successfully.\n", stdout );
  return 0;
  }


int debug_decompress( const std::string & input_filename,
                      const long long bad_pos, const int verbosity,
                      const uint8_t bad_value, const bool show_packets )
  {
  struct stat in_stats;
  const int infd = open_instream( input_filename.c_str(), &in_stats, true, true );
  if( infd < 0 ) return 1;

  Pretty_print pp( input_filename, verbosity );
  const File_index file_index( infd );
  if( file_index.retval() != 0 )
    { pp( file_index.error().c_str() ); return file_index.retval(); }

  outfd = show_packets ? -1 : STDOUT_FILENO;
  int retval = 0;
  for( long i = 0; i < file_index.members(); ++i )
    {
    const long long dpos = file_index.dblock( i ).pos();
    const long long mpos = file_index.mblock( i ).pos();
    const long long msize = file_index.mblock( i ).size();
    const unsigned dictionary_size = file_index.dictionary_size( i );
    if( verbosity >= 1 && show_packets )
      std::printf( "Decoding LZMA packets in member %ld of %ld  (mpos = %llu, msize = %llu)\n"
                   "  mpos   dpos\n",
                   i + 1, file_index.members(), mpos, msize );
    if( !isvalid_ds( dictionary_size ) )
      { show_error( "Invalid dictionary size in member header." );
        retval = 2; break; }
    uint8_t * const mbuffer = read_member( infd, mpos, msize );
    if( !mbuffer ) { retval = 1; break; }
    if( bad_pos >= 0 && file_index.mblock( i ).includes( bad_pos ) )
      {
      if( verbosity >= 1 && show_packets )
        std::printf( "Byte at pos %llu changed from 0x%02X to 0x%02X\n",
                     bad_pos, mbuffer[bad_pos-mpos], bad_value );
      mbuffer[bad_pos-mpos] = bad_value;
      }
    LZ_mtester mtester( mbuffer, msize, dictionary_size, outfd );
    const int result = mtester.debug_decode_member( dpos, mpos, show_packets );
    delete[] mbuffer;
    if( result != 0 )
      {
      if( verbosity >= 0 && result <= 2 && show_packets )
        std::printf( "%s at pos %llu\n", ( result == 2 ) ?
                     "File ends unexpectedly" : "Decoder error",
                     mpos + mtester.member_position() );
      retval = 2; break;
      }
    if( i + 1 < file_index.members() && show_packets )
      std::fputc( '\n', stdout );
    }

  retval = std::max( retval, close_outstream( &in_stats ) );
  if( verbosity >= 1 && show_packets && retval == 0 )
    std::fputs( "Done.\n", stdout );
  return retval;
  }