lzip/lzlib
1
0
Fork 0
Code Issues Activity

Adding upstream version 1.8.

Browse source 
Signed-off-by: Daniel Baumann <daniel@debian.org>
This commit is contained in:
Daniel Baumann 2025-02-20 21:21:01 +01:00
parent 0e4a2856ab
commit cadeb380f5
Signed by: daniel
GPG key ID: FBB4F0E80A80222F
28 changed files with 638 additions and 425 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
ChangeLog
View file

@ -1,3 +1,15 @@
2016-05-17 Antonio Diaz Diaz <antonio@gnu.org>
* Version 1.8 released.
* decoder.c (LZd_verify_trailer): Removed test of final code.
* main.c: Added new option '-a, --trailing-error'.
* main.c (main): Delete '--output' file if infd is a terminal.
* main.c (main): Don't use stdin more than once.
* configure: Avoid warning on some shells when testing for gcc.
* Makefile.in: Detect the existence of install-info.
* testsuite/check.sh: A POSIX shell is required to run the tests.
* testsuite/check.sh: Don't check error messages.
2015-07-08 Antonio Diaz Diaz <antonio@gnu.org>
* Version 1.7 released.
@ -153,7 +165,7 @@
* Version 0.1 released.
Copyright (C) 2009-2015 Antonio Diaz Diaz.
Copyright (C) 2009-2016 Antonio Diaz Diaz.
This file is a collection of facts, and thus it is not copyrightable,
but just in case, you have unlimited permission to copy, distribute and

4
INSTALL
View file

@ -1,7 +1,7 @@
Requirements
------------
You will need a C compiler.
I use gcc 4.9.1 and 4.1.2, but the code should compile with any
I use gcc 5.3.0 and 4.1.2, but the code should compile with any
standards compliant compiler.
Gcc is available at http://gcc.gnu.org.
@ -62,7 +62,7 @@ After running 'configure', you can run 'make' and 'make install' as
explained above.
Copyright (C) 2009-2015 Antonio Diaz Diaz.
Copyright (C) 2009-2016 Antonio Diaz Diaz.
This file is free documentation: you have unlimited permission to copy,
distribute and modify it.

10
Makefile.in
View file

@ -7,6 +7,7 @@ INSTALL_DATA = $(INSTALL) -m 644
INSTALL_DIR = $(INSTALL) -d -m 755
LDCONFIG = /sbin/ldconfig
SHELL = /bin/sh
CAN_RUN_INSTALLINFO = $(SHELL) -c "install-info --version" > /dev/null 2>&1
objs = carg_parser.o main.o
@ -109,7 +110,9 @@ install-info :
if [ ! -d "$(DESTDIR)$(infodir)" ] ; then $(INSTALL_DIR) "$(DESTDIR)$(infodir)" ; fi
-rm -f "$(DESTDIR)$(infodir)/$(pkgname).info"*
$(INSTALL_DATA) $(VPATH)/doc/$(pkgname).info "$(DESTDIR)$(infodir)/$(pkgname).info"
-install-info --info-dir="$(DESTDIR)$(infodir)" "$(DESTDIR)$(infodir)/$(pkgname).info"
-if $(CAN_RUN_INSTALLINFO) ; then \
install-info --info-dir="$(DESTDIR)$(infodir)" "$(DESTDIR)$(infodir)/$(pkgname).info" ; \
fi
install-info-compress : install-info
lzip -v -9 "$(DESTDIR)$(infodir)/$(pkgname).info"
@ -139,7 +142,9 @@ uninstall-bin :
-rm -f "$(DESTDIR)$(libdir)/lib$(libname).so.$(pkgversion)"
uninstall-info :
-install-info --info-dir="$(DESTDIR)$(infodir)" --remove "$(DESTDIR)$(infodir)/$(pkgname).info"
-if $(CAN_RUN_INSTALLINFO) ; then \
install-info --info-dir="$(DESTDIR)$(infodir)" --remove "$(DESTDIR)$(infodir)/$(pkgname).info" ; \
fi
-rm -f "$(DESTDIR)$(infodir)/$(pkgname).info"*
uninstall-man :
@ -161,6 +166,7 @@ dist : doc
$(DISTNAME)/doc/$(pkgname).texi \
$(DISTNAME)/testsuite/check.sh \
$(DISTNAME)/testsuite/test.txt \
$(DISTNAME)/testsuite/test2.txt \
$(DISTNAME)/testsuite/test.txt.lz \
$(DISTNAME)/testsuite/test_sync.lz \
$(DISTNAME)/*.h \

24
NEWS
View file

@ -1,16 +1,16 @@
Changes in version 1.7:
Changes in version 1.8:
The fast encoder, which produces a compression speed and ratio
comparable to those of gzip, has been ported from lzip.
The test of the value remaining in the range decoder has been removed.
(After extensive testing it has been found useless to detect corruption
in the decompressed data. Eliminating it reduces the number of false
positives for corruption and makes error detection more accurate).
The option "-0" has been ported from lzip to minilzip.
The option "-a, --trailing-error", which makes minilzip exit with error
status 2 if any remaining input is detected after decompressing the last
member, has been added.
If all the data to be compressed are written in advance, lzlib will
automatically adjust the header of the compressed data to use the
smallest possible dictionary size. This feature reduces the amount of
memory needed for decompression and allows minilzip to produce identical
compressed output as lzip.
When decompressing with minilzip, the file specified with the '--output'
option is now deleted if the input is a terminal.
The targets "install-compress", "install-strip-compress",
"install-info-compress" and "install-man-compress" have been added to
the Makefile.
A harmless check failure on Windows, caused by the failed comparison of
a message in text mode, has been fixed.

2
README
View file

@ -87,7 +87,7 @@ range encoding), Igor Pavlov (for putting all the above together in
LZMA), and Julian Seward (for bzip2's CLI).
Copyright (C) 2009-2015 Antonio Diaz Diaz.
Copyright (C) 2009-2016 Antonio Diaz Diaz.
This file is free documentation: you have unlimited permission to copy,
distribute and modify it.

163
bbexample.c
View file

@ -1,5 +1,5 @@
/* Buff to buff example - Test program for the lzlib library
Copyright (C) 2010-2015 Antonio Diaz Diaz.
/* Buffer to buffer example - Test program for the lzlib library
Copyright (C) 2010-2016 Antonio Diaz Diaz.
This program is free software: you have unlimited permission
to copy, distribute and modify it.
@ -11,6 +11,8 @@
compression/decompression can be implemented using lzlib.
*/
#include <errno.h>
#include <limits.h>
#ifndef __cplusplus
#include <stdbool.h>
#endif
@ -23,33 +25,102 @@
#include "lzlib.h"
/* Returns the address of a malloc'd buffer containing the file data and
its size in '*size'.
In case of error, returns 0 and does not modify '*size'.
*/
uint8_t * read_file( const char * const name, long * const size )
{
long buffer_size = 1 << 20, file_size;
uint8_t * buffer, * tmp;
FILE * const f = fopen( name, "rb" );
if( !f )
{
fprintf( stderr, "bbexample: Can't open input file '%s': %s\n",
name, strerror( errno ) );
return 0;
}
buffer = (uint8_t *)malloc( buffer_size );
if( !buffer )
{ fputs( "bbexample: Not enough memory.\n", stderr ); return 0; }
file_size = fread( buffer, 1, buffer_size, f );
while( file_size >= buffer_size )
{
if( buffer_size >= LONG_MAX )
{
fprintf( stderr, "bbexample: Input file '%s' is too large.\n", name );
free( buffer ); return 0;
}
buffer_size = ( buffer_size <= LONG_MAX / 2 ) ? 2 * buffer_size : LONG_MAX;
tmp = (uint8_t *)realloc( buffer, buffer_size );
if( !tmp )
{ fputs( "bbexample: Not enough memory.\n", stderr );
free( buffer ); return 0; }
buffer = tmp;
file_size += fread( buffer + file_size, 1, buffer_size - file_size, f );
}
if( ferror( f ) || !feof( f ) )
{
fprintf( stderr, "bbexample: Error reading file '%s': %s\n",
name, strerror( errno ) );
free( buffer ); return 0;
}
fclose( f );
*size = file_size;
return buffer;
}
/* Compresses 'size' bytes from 'data'. Returns the address of a
malloc'd buffer containing the compressed data and its size in
'*out_sizep'.
In case of error, returns 0 and does not modify '*out_sizep'.
*/
uint8_t * bbcompress( const uint8_t * const data, const int size,
int * const out_sizep )
uint8_t * bbcompress( const uint8_t * const data, const long size,
const int level, long * const out_sizep )
{
struct Lzma_options
{
int dictionary_size; /* 4 KiB .. 512 MiB */
int match_len_limit; /* 5 .. 273 */
};
/* Mapping from gzip/bzip2 style 1..9 compression modes
to the corresponding LZMA compression modes. */
const struct Lzma_options option_mapping[] =
{
{ 65535, 16 }, /* -0 (65535,16 chooses fast encoder) */
{ 1 << 20, 5 }, /* -1 */
{ 3 << 19, 6 }, /* -2 */
{ 1 << 21, 8 }, /* -3 */
{ 3 << 20, 12 }, /* -4 */
{ 1 << 22, 20 }, /* -5 */
{ 1 << 23, 36 }, /* -6 */
{ 1 << 24, 68 }, /* -7 */
{ 3 << 23, 132 }, /* -8 */
{ 1 << 25, 273 } }; /* -9 */
struct Lzma_options encoder_options;
const unsigned long long member_size = 0x7FFFFFFFFFFFFFFFULL; /* INT64_MAX */
struct LZ_Encoder * encoder;
uint8_t * new_data;
const int match_len_limit = 36;
const unsigned long long member_size = 0x7FFFFFFFFFFFFFFFULL; /* INT64_MAX */
int delta_size, new_data_size;
int new_pos = 0;
int written = 0;
const long delta_size = ( size / 4 ) + 64; /* size may be zero */
long new_data_size = delta_size; /* initial size */
long new_pos = 0;
long written = 0;
bool error = false;
int dict_size = 8 << 20; /* 8 MiB */
if( dict_size > size ) dict_size = size; /* saves memory */
if( dict_size < LZ_min_dictionary_size() )
dict_size = LZ_min_dictionary_size();
encoder = LZ_compress_open( dict_size, match_len_limit, member_size );
if( level < 0 || level > 9 ) return 0;
encoder_options = option_mapping[level];
if( encoder_options.dictionary_size > size && level != 0 )
encoder_options.dictionary_size = size; /* saves memory */
if( encoder_options.dictionary_size < LZ_min_dictionary_size() )
encoder_options.dictionary_size = LZ_min_dictionary_size();
encoder = LZ_compress_open( encoder_options.dictionary_size,
encoder_options.match_len_limit, member_size );
if( !encoder || LZ_compress_errno( encoder ) != LZ_ok )
{ LZ_compress_close( encoder ); return 0; }
delta_size = (size < 256) ? 64 : size / 4; /* size may be zero */
new_data_size = delta_size; /* initial size */
new_data = (uint8_t *)malloc( new_data_size );
if( !new_data )
{ LZ_compress_close( encoder ); return 0; }
@ -75,11 +146,12 @@ uint8_t * bbcompress( const uint8_t * const data, const int size,
if( LZ_compress_finished( encoder ) == 1 ) break;
if( new_pos >= new_data_size )
{
uint8_t * const tmp =
(uint8_t *)realloc( new_data, new_data_size + delta_size );
uint8_t * tmp;
if( new_data_size > LONG_MAX - delta_size ) { error = true; break; }
new_data_size += delta_size;
tmp = (uint8_t *)realloc( new_data, new_data_size );
if( !tmp ) { error = true; break; }
new_data = tmp;
new_data_size += delta_size;
}
}
@ -95,15 +167,15 @@ uint8_t * bbcompress( const uint8_t * const data, const int size,
'*out_sizep'.
In case of error, returns 0 and does not modify '*out_sizep'.
*/
uint8_t * bbdecompress( const uint8_t * const data, const int size,
int * const out_sizep )
uint8_t * bbdecompress( const uint8_t * const data, const long size,
long * const out_sizep )
{
struct LZ_Decoder * const decoder = LZ_decompress_open();
uint8_t * new_data;
const int delta_size = size; /* size must be > zero */
int new_data_size = delta_size; /* initial size */
int new_pos = 0;
int written = 0;
const long delta_size = size; /* size must be > zero */
long new_data_size = delta_size; /* initial size */
long new_pos = 0;
long written = 0;
bool error = false;
if( !decoder || LZ_decompress_errno( decoder ) != LZ_ok )
{ LZ_decompress_close( decoder ); return 0; }
@ -133,11 +205,12 @@ uint8_t * bbdecompress( const uint8_t * const data, const int size,
if( LZ_decompress_finished( decoder ) == 1 ) break;
if( new_pos >= new_data_size )
{
uint8_t * const tmp =
(uint8_t *)realloc( new_data, new_data_size + delta_size );
uint8_t * tmp;
if( new_data_size > LONG_MAX - delta_size ) { error = true; break; }
new_data_size += delta_size;
tmp = (uint8_t *)realloc( new_data, new_data_size );
if( !tmp ) { error = true; break; }
new_data = tmp;
new_data_size += delta_size;
}
}
@ -150,10 +223,9 @@ uint8_t * bbdecompress( const uint8_t * const data, const int size,
int main( const int argc, const char * const argv[] )
{
FILE * file;
uint8_t * in_buffer, * mid_buffer, * out_buffer;
const int in_buffer_size = 1 << 20;
int in_size, mid_size = 0, out_size = 0;
uint8_t * in_buffer;
long in_size = 0;
int level;
if( argc < 2 )
{
@ -161,29 +233,15 @@ int main( const int argc, const char * const argv[] )
return 1;
}
file = fopen( argv[1], "rb" );
if( !file )
{
fprintf( stderr, "bbexample: Can't open file '%s' for reading.\n", argv[1] );
return 1;
}
in_buffer = read_file( argv[1], &in_size );
if( !in_buffer ) return 1;
in_buffer = (uint8_t *)malloc( in_buffer_size );
if( !in_buffer )
for( level = 0; level <= 9; ++level )
{
fputs( "bbexample: Not enough memory.\n", stderr );
return 1;
}
uint8_t * mid_buffer, * out_buffer;
long mid_size = 0, out_size = 0;
in_size = fread( in_buffer, 1, in_buffer_size, file );
if( in_size >= in_buffer_size )
{
fprintf( stderr, "bbexample: Input file '%s' is too big.\n", argv[1] );
return 1;
}
fclose( file );
mid_buffer = bbcompress( in_buffer, in_size, &mid_size );
mid_buffer = bbcompress( in_buffer, in_size, level, &mid_size );
if( !mid_buffer )
{
fputs( "bbexample: Not enough memory or compress error.\n", stderr );
@ -206,6 +264,7 @@ int main( const int argc, const char * const argv[] )
free( out_buffer );
free( mid_buffer );
}
free( in_buffer );
return 0;
}

2
carg_parser.c
View file

@ -1,5 +1,5 @@
/* Arg_parser - POSIX/GNU command line argument parser. (C version)
Copyright (C) 2006-2015 Antonio Diaz Diaz.
Copyright (C) 2006-2016 Antonio Diaz Diaz.
This library is free software. Redistribution and use in source and
binary forms, with or without modification, are permitted provided

2
carg_parser.h
View file

@ -1,5 +1,5 @@
/* Arg_parser - POSIX/GNU command line argument parser. (C version)
Copyright (C) 2006-2015 Antonio Diaz Diaz.
Copyright (C) 2006-2016 Antonio Diaz Diaz.
This library is free software. Redistribution and use in source and
binary forms, with or without modification, are permitted provided

49
cbuffer.c
View file

@ -1,5 +1,5 @@
/* Lzlib - Compression library for the lzip format
Copyright (C) 2009-2015 Antonio Diaz Diaz.
Copyright (C) 2009-2016 Antonio Diaz Diaz.
This library is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -28,31 +28,32 @@
struct Circular_buffer
{
uint8_t * buffer;
int buffer_size; /* capacity == buffer_size - 1 */
int get; /* buffer is empty when get == put */
int put;
unsigned buffer_size; /* capacity == buffer_size - 1 */
unsigned get; /* buffer is empty when get == put */
unsigned put;
};
static inline void Cb_reset( struct Circular_buffer * const cb )
{ cb->get = 0; cb->put = 0; }
static inline bool Cb_init( struct Circular_buffer * const cb,
const int buf_size )
const unsigned buf_size )
{
cb->buffer = (uint8_t *)malloc( buf_size + 1 );
cb->buffer_size = buf_size + 1;
cb->get = 0;
cb->put = 0;
cb->buffer =
( cb->buffer_size > 1 ) ? (uint8_t *)malloc( cb->buffer_size ) : 0;
return ( cb->buffer != 0 );
}
static inline void Cb_free( struct Circular_buffer * const cb )
{ free( cb->buffer ); cb->buffer = 0; }
static inline int Cb_used_bytes( const struct Circular_buffer * const cb )
static inline unsigned Cb_used_bytes( const struct Circular_buffer * const cb )
{ return ( (cb->get <= cb->put) ? 0 : cb->buffer_size ) + cb->put - cb->get; }
static inline int Cb_free_bytes( const struct Circular_buffer * const cb )
static inline unsigned Cb_free_bytes( const struct Circular_buffer * const cb )
{ return ( (cb->get <= cb->put) ? cb->buffer_size : 0 ) - cb->put + cb->get - 1; }
static inline uint8_t Cb_get_byte( struct Circular_buffer * const cb )
@ -70,14 +71,25 @@ static inline void Cb_put_byte( struct Circular_buffer * const cb,
}
static bool Cb_unread_data( struct Circular_buffer * const cb,
const unsigned size )
{
if( size > Cb_free_bytes( cb ) ) return false;
if( cb->get >= size ) cb->get -= size;
else cb->get = cb->buffer_size - size + cb->get;
return true;
}
/* Copies up to 'out_size' bytes to 'out_buffer' and updates 'get'.
Returns the number of bytes copied.
*/
static int Cb_read_data( struct Circular_buffer * const cb,
uint8_t * const out_buffer, const int out_size )
static unsigned Cb_read_data( struct Circular_buffer * const cb,
uint8_t * const out_buffer,
const unsigned out_size )
{
int size = 0;
if( out_size <= 0 ) return 0;
unsigned size = 0;
if( out_size == 0 ) return 0;
if( cb->get > cb->put )
{
size = min( cb->buffer_size - cb->get, out_size );
@ -90,7 +102,7 @@ static int Cb_read_data( struct Circular_buffer * const cb,
}
if( cb->get < cb->put )
{
const int size2 = min( cb->put - cb->get, out_size - size );
const unsigned size2 = min( cb->put - cb->get, out_size - size );
if( size2 > 0 )
{
memcpy( out_buffer + size, cb->buffer + cb->get, size2 );
@ -105,11 +117,12 @@ static int Cb_read_data( struct Circular_buffer * const cb,
/* Copies up to 'in_size' bytes from 'in_buffer' and updates 'put'.
Returns the number of bytes copied.
*/
static int Cb_write_data( struct Circular_buffer * const cb,
const uint8_t * const in_buffer, const int in_size )
static unsigned Cb_write_data( struct Circular_buffer * const cb,
const uint8_t * const in_buffer,
const unsigned in_size )
{
int size = 0;
if( in_size < 0 ) return 0;
unsigned size = 0;
if( in_size == 0 ) return 0;
if( cb->put >= cb->get )
{
size = min( cb->buffer_size - cb->put - (cb->get == 0), in_size );
@ -122,7 +135,7 @@ static int Cb_write_data( struct Circular_buffer * const cb,
}
if( cb->put < cb->get )
{
const int size2 = min( cb->get - cb->put - 1, in_size - size );
const unsigned size2 = min( cb->get - cb->put - 1, in_size - size );
if( size2 > 0 )
{
memcpy( cb->buffer + cb->put, in_buffer + size, size2 );

18
configure vendored
View file

@ -1,12 +1,12 @@
#! /bin/sh
# configure script for Lzlib - Compression library for the lzip format
# Copyright (C) 2009-2015 Antonio Diaz Diaz.
# Copyright (C) 2009-2016 Antonio Diaz Diaz.
#
# This configure script is free software: you have unlimited permission
# to copy, distribute and modify it.
pkgname=lzlib
pkgversion=1.7
pkgversion=1.8
soversion=1
progname=minilzip
progname_static=${progname}
@ -34,8 +34,8 @@ CFLAGS='-Wall -W -O2'
LDFLAGS=
# checking whether we are using GNU C.
${CC} --version > /dev/null 2>&1
if [ $? != 0 ] ; then
if /bin/sh -c "${CC} --version" > /dev/null 2>&1 ; then true
else
CC=cc
CFLAGS='-W -O2'
fi
@ -74,10 +74,10 @@ while [ $# != 0 ] ; do
echo " --infodir=DIR info files directory [${infodir}]"
echo " --libdir=DIR object code libraries [${libdir}]"
echo " --mandir=DIR man pages directory [${mandir}]"
echo " --disable-static do not build a static library [enable]"
echo " --disable-static don't build a static library [enable]"
echo " (implies --enable-shared)"
echo " --enable-shared build also a shared library [disable]"
echo " --disable-ldconfig do not run ldconfig after install"
echo " --disable-ldconfig don't run ldconfig after install"
echo " CC=COMPILER C compiler to use [${CC}]"
echo " CPPFLAGS=OPTIONS command line options for the preprocessor [${CPPFLAGS}]"
echo " CFLAGS=OPTIONS command line options for the C compiler [${CFLAGS}]"
@ -163,7 +163,7 @@ if [ -z "${no_create}" ] ; then
rm -f config.status
cat > config.status << EOF
#! /bin/sh
# This file was generated automatically by configure. Do not edit.
# This file was generated automatically by configure. Don't edit.
# Run this file to recreate the current configuration.
#
# This script is free software: you have unlimited permission
@ -191,8 +191,8 @@ echo "LDFLAGS = ${LDFLAGS}"
rm -f Makefile
cat > Makefile << EOF
# Makefile for Lzlib - Compression library for the lzip format
# Copyright (C) 2009-2015 Antonio Diaz Diaz.
# This file was generated automatically by configure. Do not edit.
# Copyright (C) 2009-2016 Antonio Diaz Diaz.
# This file was generated automatically by configure. Don't edit.
#
# This Makefile is free software: you have unlimited permission
# to copy, distribute and modify it.

34
decoder.c
View file

@ -1,5 +1,5 @@
/* Lzlib - Compression library for the lzip format
Copyright (C) 2009-2015 Antonio Diaz Diaz.
Copyright (C) 2009-2016 Antonio Diaz Diaz.
This library is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -28,28 +28,29 @@
static bool LZd_verify_trailer( struct LZ_decoder * const d )
{
File_trailer trailer;
const unsigned long long member_size = d->rdec->member_position + Ft_size;
int size = Rd_read_data( d->rdec, trailer, Ft_size );
if( size < Ft_size )
return false;
return ( d->rdec->code == 0 &&
Ft_get_data_crc( trailer ) == LZd_crc( d ) &&
return ( Ft_get_data_crc( trailer ) == LZd_crc( d ) &&
Ft_get_data_size( trailer ) == LZd_data_position( d ) &&
Ft_get_member_size( trailer ) == member_size );
Ft_get_member_size( trailer ) == d->rdec->member_position );
}
/* Return value: 0 = OK, 1 = decoder error, 2 = unexpected EOF,
3 = trailer error, 4 = unknown marker found. */
3 = trailer error, 4 = unknown marker found,
5 = library error. */
static int LZd_decode_member( struct LZ_decoder * const d )
{
struct Range_decoder * const rdec = d->rdec;
State * const state = &d->state;
/* unsigned long long old_mpos = d->rdec->member_position; */
if( d->member_finished ) return 0;
if( !Rd_try_reload( rdec, false ) ) return 0;
if( !Rd_try_reload( rdec, false ) )
{ if( !rdec->at_stream_end ) return 0; else return 2; }
if( d->verify_trailer_pending )
{
if( Rd_available_bytes( rdec ) < Ft_size && !rdec->at_stream_end )
@ -62,8 +63,12 @@ static int LZd_decode_member( struct LZ_decoder * const d )
while( !Rd_finished( rdec ) )
{
const int pos_state = LZd_data_position( d ) & pos_state_mask;
if( !Rd_enough_available_bytes( rdec ) || !LZd_enough_free_bytes( d ) )
return 0;
/* const unsigned long long mpos = d->rdec->member_position;
if( mpos - old_mpos > rd_min_available_bytes ) return 5;
old_mpos = mpos; */
if( !Rd_enough_available_bytes( rdec ) ) /* check unexpected eof */
{ if( !rdec->at_stream_end ) return 0; else break; }
if( !LZd_enough_free_bytes( d ) ) return 0;
if( Rd_decode_bit( rdec, &d->bm_match[*state][pos_state] ) == 0 ) /* 1st bit */
{
const uint8_t prev_byte = LZd_peek_prev( d );
@ -113,8 +118,8 @@ static int LZd_decode_member( struct LZ_decoder * const d )
}
else /* match */
{
int dis_slot;
const unsigned rep0_saved = d->rep0;
int dis_slot;
len = min_match_len + Rd_decode_len( rdec, &d->match_len_model, pos_state );
dis_slot = Rd_decode_tree6( rdec, d->bm_dis_slot[get_len_state(len)] );
if( dis_slot < start_dis_model ) d->rep0 = dis_slot;
@ -142,8 +147,8 @@ static int LZd_decode_member( struct LZ_decoder * const d )
}
if( len == min_match_len + 1 ) /* Sync Flush marker */
{
if( Rd_try_reload( rdec, true ) ) continue;
else return 0;
if( Rd_try_reload( rdec, true ) ) { /*old_mpos += 5;*/ continue; }
else { if( !rdec->at_stream_end ) return 0; else break; }
}
return 4;
}
@ -151,7 +156,8 @@ static int LZd_decode_member( struct LZ_decoder * const d )
}
d->rep3 = d->rep2; d->rep2 = d->rep1; d->rep1 = rep0_saved;
*state = St_set_match( *state );
if( d->rep0 >= d->dictionary_size || d->rep0 >= LZd_data_position( d ) )
if( d->rep0 >= d->dictionary_size ||
( d->rep0 >= d->cb.put && !d->pos_wrapped ) )
return 1;
}
LZd_copy_block( d, d->rep0, len );

97
decoder.h
View file

@ -1,5 +1,5 @@
/* Lzlib - Compression library for the lzip format
Copyright (C) 2009-2015 Antonio Diaz Diaz.
Copyright (C) 2009-2016 Antonio Diaz Diaz.
This library is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -58,34 +58,40 @@ static inline void Rd_finish( struct Range_decoder * const rdec )
{ rdec->at_stream_end = true; }
static inline bool Rd_enough_available_bytes( const struct Range_decoder * const rdec )
{
return ( Cb_used_bytes( &rdec->cb ) >= rd_min_available_bytes ||
( rdec->at_stream_end && Cb_used_bytes( &rdec->cb ) > 0 ) );
}
{ return ( Cb_used_bytes( &rdec->cb ) >= rd_min_available_bytes ); }
static inline int Rd_available_bytes( const struct Range_decoder * const rdec )
static inline unsigned Rd_available_bytes( const struct Range_decoder * const rdec )
{ return Cb_used_bytes( &rdec->cb ); }
static inline int Rd_free_bytes( const struct Range_decoder * const rdec )
static inline unsigned Rd_free_bytes( const struct Range_decoder * const rdec )
{ if( rdec->at_stream_end ) return 0; return Cb_free_bytes( &rdec->cb ); }
static inline void Rd_purge( struct Range_decoder * const rdec )
{ rdec->at_stream_end = true; Cb_reset( &rdec->cb ); }
static inline unsigned long long Rd_purge( struct Range_decoder * const rdec )
{
const unsigned long long size =
rdec->member_position + Cb_used_bytes( &rdec->cb );
Cb_reset( &rdec->cb );
rdec->member_position = 0; rdec->at_stream_end = true;
return size;
}
static inline void Rd_reset( struct Range_decoder * const rdec )
{ rdec->at_stream_end = false; Cb_reset( &rdec->cb ); }
{ Cb_reset( &rdec->cb );
rdec->member_position = 0; rdec->at_stream_end = false; }
/* Seeks a member header and updates 'get'.
Returns true if it finds a valid header.
/* Seeks a member header and updates 'get'. '*skippedp' is set to the
number of bytes skipped. Returns true if it finds a valid header.
*/
static bool Rd_find_header( struct Range_decoder * const rdec )
static bool Rd_find_header( struct Range_decoder * const rdec,
int * const skippedp )
{
*skippedp = 0;
while( rdec->cb.get != rdec->cb.put )
{
if( rdec->cb.buffer[rdec->cb.get] == magic_string[0] )
{
int get = rdec->cb.get;
unsigned get = rdec->cb.get;
int i;
File_header header;
for( i = 0; i < Fh_size; ++i )
@ -97,36 +103,12 @@ static bool Rd_find_header( struct Range_decoder * const rdec )
if( Fh_verify( header ) ) return true;
}
if( ++rdec->cb.get >= rdec->cb.buffer_size ) rdec->cb.get = 0;
++*skippedp;
}
return false;
}
/* Returns true, fills 'header', and updates 'get' if 'get' points to a
valid header.
Else returns false and leaves 'get' unmodified.
*/
static bool Rd_read_header( struct Range_decoder * const rdec,
File_header header )
{
int get = rdec->cb.get;
int i;
for( i = 0; i < Fh_size; ++i )
{
if( get == rdec->cb.put ) return false; /* not enough data */
header[i] = rdec->cb.buffer[get];
if( ++get >= rdec->cb.buffer_size ) get = 0;
}
if( Fh_verify( header ) )
{
rdec->cb.get = get;
rdec->member_position = Fh_size;
rdec->reload_pending = true;
return true;
}
return false;
}
static inline int Rd_write_data( struct Range_decoder * const rdec,
const uint8_t * const inbuf, const int size )
{
@ -148,6 +130,15 @@ static inline int Rd_read_data( struct Range_decoder * const rdec,
return sz;
}
static inline bool Rd_unread_data( struct Range_decoder * const rdec,
const unsigned size )
{
if( size > rdec->member_position || !Cb_unread_data( &rdec->cb, size ) )
return false;
rdec->member_position -= size;
return true;
}
static bool Rd_try_reload( struct Range_decoder * const rdec, const bool force )
{
if( force ) rdec->reload_pending = true;
@ -314,6 +305,7 @@ struct LZ_decoder
uint32_t crc;
bool member_finished;
bool verify_trailer_pending;
bool pos_wrapped;
unsigned rep0; /* rep[0-3] latest four distances */
unsigned rep1; /* used for efficient coding of */
unsigned rep2; /* repeated distances */
@ -340,15 +332,15 @@ static inline bool LZd_enough_free_bytes( const struct LZ_decoder * const d )
static inline uint8_t LZd_peek_prev( const struct LZ_decoder * const d )
{
const int i = ( ( d->cb.put > 0 ) ? d->cb.put : d->cb.buffer_size ) - 1;
const unsigned i = ( ( d->cb.put > 0 ) ? d->cb.put : d->cb.buffer_size ) - 1;
return d->cb.buffer[i];
}
static inline uint8_t LZd_peek( const struct LZ_decoder * const d,
const int distance )
const unsigned distance )
{
int i = d->cb.put - distance - 1;
if( i < 0 ) i += d->cb.buffer_size;
unsigned i = d->cb.put - distance - 1;
if( d->cb.put <= distance ) i += d->cb.buffer_size;
return d->cb.buffer[i];
}
@ -357,16 +349,20 @@ static inline void LZd_put_byte( struct LZ_decoder * const d, const uint8_t b )
CRC32_update_byte( &d->crc, b );
d->cb.buffer[d->cb.put] = b;
if( ++d->cb.put >= d->cb.buffer_size )
{ d->partial_data_pos += d->cb.put; d->cb.put = 0; }
{ d->partial_data_pos += d->cb.put; d->cb.put = 0; d->pos_wrapped = true; }
}
static inline void LZd_copy_block( struct LZ_decoder * const d,
const int distance, int len )
const unsigned distance, unsigned len )
{
int i = d->cb.put - distance - 1;
if( i < 0 ) i += d->cb.buffer_size;
if( len < d->cb.buffer_size - max( d->cb.put, i ) &&
len <= abs( d->cb.put - i ) ) /* no wrap, no overlap */
unsigned i = d->cb.put - distance - 1;
bool fast;
if( d->cb.put <= distance )
{ i += d->cb.buffer_size;
fast = ( len <= d->cb.buffer_size - i && len <= i - d->cb.put ); }
else
fast = ( len < d->cb.buffer_size - d->cb.put && len <= d->cb.put - i );
if( fast ) /* no wrap, no overlap */
{
CRC32_update_buf( &d->crc, d->cb.buffer + i, len );
memcpy( d->cb.buffer + d->cb.put, d->cb.buffer + i, len );
@ -381,7 +377,7 @@ static inline void LZd_copy_block( struct LZ_decoder * const d,
static inline bool LZd_init( struct LZ_decoder * const d,
struct Range_decoder * const rde,
const int dict_size )
const unsigned dict_size )
{
if( !Cb_init( &d->cb, max( 65536, dict_size ) + lzd_min_free_bytes ) )
return false;
@ -391,6 +387,7 @@ static inline bool LZd_init( struct LZ_decoder * const d,
d->crc = 0xFFFFFFFFU;
d->member_finished = false;
d->verify_trailer_pending = false;
d->pos_wrapped = false;
d->rep0 = 0;
d->rep1 = 0;
d->rep2 = 0;

51
doc/lzlib.info
View file

@ -11,7 +11,7 @@ File: lzlib.info, Node: Top, Next: Introduction, Up: (dir)
Lzlib Manual
************
This manual is for Lzlib (version 1.7, 8 July 2015).
This manual is for Lzlib (version 1.8, 17 May 2016).
* Menu:
@ -29,7 +29,7 @@ This manual is for Lzlib (version 1.7, 8 July 2015).
* Concept index:: Index of concepts
Copyright (C) 2009-2015 Antonio Diaz Diaz.
Copyright (C) 2009-2016 Antonio Diaz Diaz.
This manual is free documentation: you have unlimited permission to
copy, distribute and modify it.
@ -53,7 +53,7 @@ availability:
recovery means. The lziprecover program can repair bit-flip errors
(one of the most common forms of data corruption) in lzip files,
and provides data recovery capabilities, including error-checked
merging of damaged copies of a file. *note Data safety:
merging of damaged copies of a file. *Note Data safety:
(lziprecover)Data safety.
* The lzip format is as simple as possible (but not simpler). The
@ -236,7 +236,7 @@ calling 'LZ_compress_errno' before using it.
Valid values range from 4 KiB to 512 MiB. Note that dictionary
sizes are quantized. If the specified size does not match one of
the valid sizes, it will be rounded upwards by adding up to
(DICTIONARY_SIZE / 16) to it.
(DICTIONARY_SIZE / 8) to it.
MATCH_LEN_LIMIT sets the match length limit in bytes. Valid values
range from 5 to 273. Larger values usually give better compression
@ -270,7 +270,7 @@ calling 'LZ_compress_errno' before using it.
-- Function: int LZ_compress_restart_member ( struct LZ_Encoder *
const ENCODER, const unsigned long long MEMBER_SIZE )
Use this function to start a new member, in a multi-member data
Use this function to start a new member in a multimember data
stream. Call this function only after
'LZ_compress_member_finished' indicates that the current member
has been fully read (with the 'LZ_compress_read' function).
@ -325,8 +325,8 @@ calling 'LZ_compress_errno' before using it.
-- Function: int LZ_compress_member_finished ( struct LZ_Encoder *
const ENCODER )
Returns 1 if the current member, in a multi-member data stream,
has been fully read and 'LZ_compress_restart_member' can be safely
Returns 1 if the current member, in a multimember data stream, has
been fully read and 'LZ_compress_restart_member' can be safely
called. Otherwise it returns 0.
-- Function: unsigned long long LZ_compress_data_position ( struct
@ -492,7 +492,7 @@ this return value only tells you that an error has occurred. To find out
what kind of error it was, you need to verify the error code by calling
'LZ_(de)compress_errno'.
Library functions do not change the value returned by
Library functions don't change the value returned by
'LZ_(de)compress_errno' when they succeed; thus, the value returned by
'LZ_(de)compress_errno' after a successful call is not necessarily
LZ_ok, and you should not use 'LZ_(de)compress_errno' to determine
@ -520,8 +520,9 @@ whether a call failed. If the call failed, then you can examine
finished.
-- Constant: enum LZ_Errno LZ_header_error
Reading of member header failed. If this happens at the end of the
data stream it may indicate trailing garbage.
An invalid member header (one with the wrong magic bytes) was
read. If this happens at the end of the data stream it may
indicate trailing data.
-- Constant: enum LZ_Errno LZ_unexpected_eof
The end of the data stream was reached in the middle of a member.
@ -579,12 +580,12 @@ with no additional information before, between, or after them.
Each member has the following structure:
+--+--+--+--+----+----+=============+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| ID string | VN | DS | Lzma stream | CRC32 | Data size | Member size |
| ID string | VN | DS | LZMA stream | CRC32 | Data size | Member size |
+--+--+--+--+----+----+=============+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
All multibyte values are stored in little endian order.
'ID string'
'ID string (the "magic" bytes)'
A four byte string, identifying the lzip format, with the value
"LZIP" (0x4C, 0x5A, 0x49, 0x50).
@ -602,8 +603,8 @@ with no additional information before, between, or after them.
Example: 0xD3 = 2^19 - 6 * 2^15 = 512 KiB - 6 * 32 KiB = 320 KiB
Valid values for dictionary size range from 4 KiB to 512 MiB.
'Lzma stream'
The lzma stream, finished by an end of stream marker. Uses default
'LZMA stream'
The LZMA stream, finished by an end of stream marker. Uses default
values for encoder properties. *Note Stream format: (lzip)Stream
format, for a complete description.
Lzip only uses the LZMA marker '2' ("End Of Stream" marker). Lzlib
@ -619,7 +620,7 @@ with no additional information before, between, or after them.
Total size of the member, including header and trailer. This field
acts as a distributed index, allows the verification of stream
integrity, and facilitates safe recovery of undamaged members from
multi-member files.
multimember files.

@ -688,7 +689,7 @@ Example 4: Decompression using LZ_decompress_write_size.
7) LZ_decompress_close
Example 5: Multi-member compression (MEMBER_SIZE < total output).
Example 5: Multimember compression (MEMBER_SIZE < total output).
1) LZ_compress_open
2) go to step 5 if LZ_compress_write_size returns 0
@ -702,7 +703,7 @@ Example 5: Multi-member compression (MEMBER_SIZE < total output).
10) LZ_compress_close
Example 6: Multi-member compression (user-restarted members).
Example 6: Multimember compression (user-restarted members).
1) LZ_compress_open
2) LZ_compress_write
@ -718,7 +719,7 @@ Example 6: Multi-member compression (user-restarted members).
12) LZ_compress_close
Example 7: Decompression with automatic removal of leading garbage.
Example 7: Decompression with automatic removal of leading data.
1) LZ_decompress_open
2) LZ_decompress_sync_to_member
@ -790,13 +791,13 @@ Node: Library version5918
Node: Buffering6563
Node: Parameter limits7783
Node: Compression functions8742
Node: Decompression functions15286
Node: Error codes21454
Node: Error messages23393
Node: Data format23972
Node: Examples26518
Node: Problems30604
Node: Concept index31176
Node: Decompression functions15282
Node: Error codes21450
Node: Error messages23425
Node: Data format24004
Node: Examples26569
Node: Problems30650
Node: Concept index31222

End Tag Table

37
doc/lzlib.texi
View file

@ -6,8 +6,8 @@
@finalout
@c %**end of header
@set UPDATED 8 July 2015
@set VERSION 1.7
@set UPDATED 17 May 2016
@set VERSION 1.8
@dircategory Data Compression
@direntry
@ -50,7 +50,7 @@ This manual is for Lzlib (version @value{VERSION}, @value{UPDATED}).
@end menu
@sp 1
Copyright @copyright{} 2009-2015 Antonio Diaz Diaz.
Copyright @copyright{} 2009-2016 Antonio Diaz Diaz.
This manual is free documentation: you have unlimited permission
to copy, distribute and modify it.
@ -78,7 +78,7 @@ program can repair bit-flip errors (one of the most common forms of data
corruption) in lzip files, and provides data recovery capabilities,
including error-checked merging of damaged copies of a file.
@ifnothtml
@ref{Data safety,,,lziprecover}.
@xref{Data safety,,,lziprecover}.
@end ifnothtml
@item
@ -269,7 +269,7 @@ should be freed with @samp{LZ_compress_close} to avoid memory leaks.
@var{dictionary_size} sets the dictionary size to be used, in bytes.
Valid values range from 4 KiB to 512 MiB. Note that dictionary sizes are
quantized. If the specified size does not match one of the valid sizes,
it will be rounded upwards by adding up to (@var{dictionary_size} / 16)
it will be rounded upwards by adding up to (@var{dictionary_size} / 8)
to it.
@var{match_len_limit} sets the match length limit in bytes. Valid values
@ -307,7 +307,7 @@ After all the produced compressed data have been read with
@deftypefun int LZ_compress_restart_member ( struct LZ_Encoder * const @var{encoder}, const unsigned long long @var{member_size} )
Use this function to start a new member, in a multi-member data stream.
Use this function to start a new member in a multimember data stream.
Call this function only after @samp{LZ_compress_member_finished}
indicates that the current member has been fully read (with the
@samp{LZ_compress_read} function).
@ -370,7 +370,7 @@ can be safely called. Otherwise it returns 0.
@deftypefun int LZ_compress_member_finished ( struct LZ_Encoder * const @var{encoder} )
Returns 1 if the current member, in a multi-member data stream, has been
Returns 1 if the current member, in a multimember data stream, has been
fully read and @samp{LZ_compress_restart_member} can be safely called.
Otherwise it returns 0.
@end deftypefun
@ -560,7 +560,7 @@ this return value only tells you that an error has occurred. To find out
what kind of error it was, you need to verify the error code by calling
@samp{LZ_(de)compress_errno}.
Library functions do not change the value returned by
Library functions don't change the value returned by
@samp{LZ_(de)compress_errno} when they succeed; thus, the value returned
by @samp{LZ_(de)compress_errno} after a successful call is not
necessarily LZ_ok, and you should not use @samp{LZ_(de)compress_errno}
@ -592,8 +592,9 @@ finished.
@end deftypevr
@deftypevr Constant {enum LZ_Errno} LZ_header_error
Reading of member header failed. If this happens at the end of the data
stream it may indicate trailing garbage.
An invalid member header (one with the wrong magic bytes) was read. If
this happens at the end of the data stream it may indicate trailing
data.
@end deftypevr
@deftypevr Constant {enum LZ_Errno} LZ_unexpected_eof
@ -657,14 +658,14 @@ with no additional information before, between, or after them.
Each member has the following structure:
@verbatim
+--+--+--+--+----+----+=============+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| ID string | VN | DS | Lzma stream | CRC32 | Data size | Member size |
| ID string | VN | DS | LZMA stream | CRC32 | Data size | Member size |
+--+--+--+--+----+----+=============+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@end verbatim
All multibyte values are stored in little endian order.
@table @samp
@item ID string
@item ID string (the "magic" bytes)
A four byte string, identifying the lzip format, with the value "LZIP"
(0x4C, 0x5A, 0x49, 0x50).
@ -681,8 +682,8 @@ from the base size to obtain the dictionary size.@*
Example: 0xD3 = 2^19 - 6 * 2^15 = 512 KiB - 6 * 32 KiB = 320 KiB@*
Valid values for dictionary size range from 4 KiB to 512 MiB.
@item Lzma stream
The lzma stream, finished by an end of stream marker. Uses default
@item LZMA stream
The LZMA stream, finished by an end of stream marker. Uses default
values for encoder properties.
@ifnothtml
@xref{Stream format,,,lzip},
@ -704,7 +705,7 @@ Size of the uncompressed original data.
@item Member size (8 bytes)
Total size of the member, including header and trailer. This field acts
as a distributed index, allows the verification of stream integrity, and
facilitates safe recovery of undamaged members from multi-member files.
facilitates safe recovery of undamaged members from multimember files.
@end table
@ -785,7 +786,7 @@ Example 4: Decompression using LZ_decompress_write_size.
@sp 1
@noindent
Example 5: Multi-member compression (@var{member_size} < total output).
Example 5: Multimember compression (@var{member_size} < total output).
@example
1) LZ_compress_open
@ -802,7 +803,7 @@ Example 5: Multi-member compression (@var{member_size} < total output).
@sp 1
@noindent
Example 6: Multi-member compression (user-restarted members).
Example 6: Multimember compression (user-restarted members).
@example
1) LZ_compress_open
@ -821,7 +822,7 @@ Example 6: Multi-member compression (user-restarted members).
@sp 1
@noindent
Example 7: Decompression with automatic removal of leading garbage.
Example 7: Decompression with automatic removal of leading data.
@example
1) LZ_decompress_open

22
doc/minilzip.1
View file

@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.46.1.
.TH MINILZIP "1" "July 2015" "minilzip 1.7" "User Commands"
.TH MINILZIP "1" "May 2016" "minilzip 1.8" "User Commands"
.SH NAME
minilzip \- reduces the size of files
.SH SYNOPSIS
@ -15,11 +15,14 @@ display this help and exit
\fB\-V\fR, \fB\-\-version\fR
output version information and exit
.TP
\fB\-a\fR, \fB\-\-trailing\-error\fR
exit with error status if trailing data
.TP
\fB\-b\fR, \fB\-\-member\-size=\fR<bytes>
set member size limit in bytes
.TP
\fB\-c\fR, \fB\-\-stdout\fR
send output to standard output
write to standard output, keep input files
.TP
\fB\-d\fR, \fB\-\-decompress\fR
decompress
@ -37,7 +40,7 @@ keep (don't delete) input files
set match length limit in bytes [36]
.TP
\fB\-o\fR, \fB\-\-output=\fR<file>
if reading stdin, place the output into <file>
if reading standard input, write to <file>
.TP
\fB\-q\fR, \fB\-\-quiet\fR
suppress all messages
@ -63,13 +66,16 @@ alias for \fB\-0\fR
\fB\-\-best\fR
alias for \fB\-9\fR
.PP
If no file names are given, minilzip compresses or decompresses
from standard input to standard output.
If no file names are given, or if a file is '\-', minilzip compresses or
decompresses from standard input to standard output.
Numbers may be followed by a multiplier: k = kB = 10^3 = 1000,
Ki = KiB = 2^10 = 1024, M = 10^6, Mi = 2^20, G = 10^9, Gi = 2^30, etc...
Dictionary sizes 12 to 29 are interpreted as powers of two, meaning 2^12
to 2^29 bytes.
.PP
The bidimensional parameter space of LZMA can't be mapped to a linear
scale optimal for all files. If your files are large, very repetitive,
etc, you may need to use the \fB\-\-match\-length\fR and \fB\-\-dictionary\-size\fR
etc, you may need to use the \fB\-\-dictionary\-size\fR and \fB\-\-match\-length\fR
options directly to achieve optimal performance.
.PP
Exit status: 0 for a normal exit, 1 for environmental problems (file
@ -81,8 +87,8 @@ Report bugs to lzip\-bug@nongnu.org
.br
Lzlib home page: http://www.nongnu.org/lzip/lzlib.html
.SH COPYRIGHT
Copyright \(co 2015 Antonio Diaz Diaz.
Using lzlib 1.7
Copyright \(co 2016 Antonio Diaz Diaz.
Using lzlib 1.8
License GPLv2+: GNU GPL version 2 or later <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.

21
encoder.c
View file

@ -1,5 +1,5 @@
/* Lzlib - Compression library for the lzip format
Copyright (C) 2009-2015 Antonio Diaz Diaz.
Copyright (C) 2009-2016 Antonio Diaz Diaz.
This library is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -37,7 +37,7 @@ static int LZe_get_match_pairs( struct LZ_encoder * const e, struct Pair * pairs
const int min_pos = ( e->eb.mb.pos > e->eb.mb.dictionary_size ) ?
e->eb.mb.pos - e->eb.mb.dictionary_size : 0;
const uint8_t * const data = Mb_ptr_to_current_pos( &e->eb.mb );
int count, delta, key2, key3, key4, newpos;
int count, key2, key3, key4, newpos;
unsigned tmp;
int len_limit = e->match_len_limit;
@ -74,7 +74,7 @@ static int LZe_get_match_pairs( struct LZ_encoder * const e, struct Pair * pairs
}
if( num_pairs > 0 )
{
delta = pos1 - np2;
const int delta = pos1 - np2;
while( maxlen < len_limit && data[maxlen-delta] == data[maxlen] )
++maxlen;
pairs[num_pairs-1].len = maxlen;
@ -90,6 +90,7 @@ static int LZe_get_match_pairs( struct LZ_encoder * const e, struct Pair * pairs
for( count = e->cycles; ; )
{
int delta;
if( newpos <= min_pos || --count < 0 ) { *ptr0 = *ptr1 = 0; break; }
if( e->been_flushed ) len = 0;
@ -195,16 +196,16 @@ static int LZe_sequence_optimizer( struct LZ_encoder * const e,
}
if( replens[rep_index] >= e->match_len_limit )
{
e->trials[0].dis = rep_index;
e->trials[0].price = replens[rep_index];
e->trials[0].dis = rep_index;
if( !LZe_move_and_update( e, replens[rep_index] ) ) return 0;
return replens[rep_index];
}
if( main_len >= e->match_len_limit )
{
e->trials[0].dis = e->pairs[num_pairs-1].dis + num_rep_distances;
e->trials[0].price = main_len;
e->trials[0].dis = e->pairs[num_pairs-1].dis + num_rep_distances;
if( !LZe_move_and_update( e, main_len ) ) return 0;
return main_len;
}
@ -217,13 +218,12 @@ static int LZe_sequence_optimizer( struct LZ_encoder * const e,
const uint8_t cur_byte = Mb_peek( &e->eb.mb, 0 );
const uint8_t match_byte = Mb_peek( &e->eb.mb, reps[0] + 1 );
e->trials[0].state = state;
e->trials[1].dis = -1; /* literal */
e->trials[1].price = price0( e->eb.bm_match[state][pos_state] );
if( St_is_char( state ) )
e->trials[1].price += LZeb_price_literal( &e->eb, prev_byte, cur_byte );
else
e->trials[1].price += LZeb_price_matched( &e->eb, prev_byte, cur_byte, match_byte );
e->trials[1].dis = -1; /* literal */
if( match_byte == cur_byte )
Tr_update( &e->trials[1], rep_match_price +
@ -233,16 +233,15 @@ static int LZe_sequence_optimizer( struct LZ_encoder * const e,
if( num_trials < min_match_len )
{
e->trials[0].dis = e->trials[1].dis;
e->trials[0].price = 1;
e->trials[0].dis = e->trials[1].dis;
if( !Mb_move_pos( &e->eb.mb ) ) return 0;
return 1;
}
e->trials[0].state = state;
for( i = 0; i < num_rep_distances; ++i )
e->trials[0].reps[i] = reps[i];
e->trials[1].prev_index = 0;
e->trials[1].prev_index2 = single_step_trial;
for( len = min_match_len; len <= num_trials; ++len )
e->trials[len].price = infinite_price;
@ -556,8 +555,8 @@ static bool LZe_encode_member( struct LZ_encoder * const e )
{
const int pos_state =
( Mb_data_position( &e->eb.mb ) - ahead ) & pos_state_mask;
const int dis = e->trials[i].dis;
const int len = e->trials[i].price;
const int dis = e->trials[i].dis;
bool bit = ( dis < 0 );
Re_encode_bit( &e->eb.renc, &e->eb.bm_match[*state][pos_state], !bit );

4
encoder.h
View file

@ -1,5 +1,5 @@
/* Lzlib - Compression library for the lzip format
Copyright (C) 2009-2015 Antonio Diaz Diaz.
Copyright (C) 2009-2016 Antonio Diaz Diaz.
This library is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -322,6 +322,8 @@ static inline bool LZe_init( struct LZ_encoder * const e,
Lp_init( &e->rep_len_prices, &e->eb.rep_len_model, e->match_len_limit );
e->pending_num_pairs = 0;
e->num_dis_slots = 2 * real_bits( e->eb.mb.dictionary_size - 1 );
e->trials[1].prev_index = 0;
e->trials[1].prev_index2 = single_step_trial;
e->price_counter = 0;
e->dis_price_counter = 0;
e->align_price_counter = 0;

9
encoder_base.c
View file

@ -1,5 +1,5 @@
/* Lzlib - Compression library for the lzip format
Copyright (C) 2009-2015 Antonio Diaz Diaz.
Copyright (C) 2009-2016 Antonio Diaz Diaz.
This library is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -91,7 +91,7 @@ static bool Mb_init( struct Matchfinder_base * const mb,
}
static void Mb_adjust_distionary_size( struct Matchfinder_base * const mb )
static void Mb_adjust_dictionary_size( struct Matchfinder_base * const mb )
{
if( mb->stream_pos < mb->dictionary_size )
{
@ -133,7 +133,7 @@ static bool LZeb_full_flush( struct LZ_encoder_base * const eb )
const State state = eb->state;
File_trailer trailer;
if( eb->member_finished ||
Cb_free_bytes( &eb->renc.cb ) < max_marker_size + Ft_size )
Cb_free_bytes( &eb->renc.cb ) < max_marker_size + eb->renc.ff_count + Ft_size )
return false;
Re_encode_bit( &eb->renc, &eb->bm_match[state][pos_state], 1 );
Re_encode_bit( &eb->renc, &eb->bm_rep[state], 0 );
@ -154,7 +154,8 @@ static bool LZeb_sync_flush( struct LZ_encoder_base * const eb )
int i;
const int pos_state = Mb_data_position( &eb->mb ) & pos_state_mask;
const State state = eb->state;
if( eb->member_finished || Cb_free_bytes( &eb->renc.cb ) < 2 * max_marker_size )
if( eb->member_finished ||
Cb_free_bytes( &eb->renc.cb ) < (2 * max_marker_size) + eb->renc.ff_count )
return false;
for( i = 0; i < 2; ++i ) /* 2 consecutive markers guarantee decoding */
{

10
encoder_base.h
View file

@ -1,5 +1,5 @@
/* Lzlib - Compression library for the lzip format
Copyright (C) 2009-2015 Antonio Diaz Diaz.
Copyright (C) 2009-2016 Antonio Diaz Diaz.
This library is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -306,7 +306,7 @@ static inline bool Mb_move_pos( struct Matchfinder_base * const mb )
struct Range_encoder
{
struct Circular_buffer cb;
int min_free_bytes;
unsigned min_free_bytes;
uint64_t low;
unsigned long long partial_member_pos;
uint32_t range;
@ -344,7 +344,7 @@ static inline void Re_reset( struct Range_encoder * const renc )
static inline bool Re_init( struct Range_encoder * const renc,
const unsigned dictionary_size,
const int min_free_bytes )
const unsigned min_free_bytes )
{
if( !Cb_init( &renc->cb, 65536 + min_free_bytes ) ) return false;
renc->min_free_bytes = min_free_bytes;
@ -362,7 +362,7 @@ Re_member_position( const struct Range_encoder * const renc )
{ return renc->partial_member_pos + Cb_used_bytes( &renc->cb ) + renc->ff_count; }
static inline bool Re_enough_free_bytes( const struct Range_encoder * const renc )
{ return Cb_free_bytes( &renc->cb ) >= renc->min_free_bytes; }
{ return Cb_free_bytes( &renc->cb ) >= renc->min_free_bytes + renc->ff_count; }
static inline int Re_read_data( struct Range_encoder * const renc,
uint8_t * const out_buffer, const int out_size )
@ -518,7 +518,7 @@ static inline bool LZeb_init( struct LZ_encoder_base * const eb,
const int after_size, const int dict_factor,
const int num_prev_positions23,
const int pos_array_factor,
const int min_free_bytes,
const unsigned min_free_bytes,
const unsigned long long member_size )
{
if( !Mb_init( &eb->mb, before, dict_size, after_size, dict_factor,

6
fast_encoder.c
View file

@ -1,5 +1,5 @@
/* Lzlib - Compression library for the lzip format
Copyright (C) 2009-2015 Antonio Diaz Diaz.
Copyright (C) 2009-2016 Antonio Diaz Diaz.
This library is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -40,7 +40,6 @@ int FLZe_longest_match_len( struct FLZ_encoder * const fe, int * const distance
newpos = fe->eb.mb.prev_positions[fe->key4];
fe->eb.mb.prev_positions[fe->key4] = pos1;
for( count = 4; ; )
{
if( --count < 0 || newpos <= 0 ) { *ptr0 = 0; break; }
@ -106,12 +105,11 @@ bool FLZe_encode_member( struct FLZ_encoder * const fe )
Re_member_position( &fe->eb.renc ) < fe->eb.member_size_limit )
{
int match_distance;
int main_len, pos_state, len;
int main_len, pos_state, len = 0;
if( !Mb_enough_available_bytes( &fe->eb.mb ) ||
!Re_enough_free_bytes( &fe->eb.renc ) ) return true;
main_len = FLZe_longest_match_len( fe, &match_distance );
pos_state = Mb_data_position( &fe->eb.mb ) & pos_state_mask;
len = 0;
for( i = 0; i < num_rep_distances; ++i )
{

2
fast_encoder.h
View file

@ -1,5 +1,5 @@
/* Lzlib - Compression library for the lzip format
Copyright (C) 2009-2015 Antonio Diaz Diaz.
Copyright (C) 2009-2016 Antonio Diaz Diaz.
This library is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by

10
lzcheck.c
View file

@ -1,5 +1,5 @@
/* Lzcheck - Test program for the lzlib library
Copyright (C) 2009-2015 Antonio Diaz Diaz.
Copyright (C) 2009-2016 Antonio Diaz Diaz.
This program is free software: you have unlimited permission
to copy, distribute and modify it.
@ -95,7 +95,7 @@ int lzcheck( FILE * const file, const int dictionary_size )
if( out_size != in_size || memcmp( in_buffer + l, out_buffer, out_size ) )
{
fprintf( stderr, "lzcheck: Sync error at pos %d. in_size = %d, out_size = %d.\n",
fprintf( stderr, "lzcheck: Sync error at pos %d in_size = %d, out_size = %d\n",
l, in_size, out_size );
for( i = 0; i < in_size; ++i )
fputc( in_buffer[l+i], stderr );
@ -167,8 +167,8 @@ int lzcheck( FILE * const file, const int dictionary_size )
if( out_size != in_size || memcmp( in_buffer + l, out_buffer, out_size ) )
{
fprintf( stderr, "lzcheck: Sync error at pos %d. in_size = %d, out_size = %d.\n",
l, in_size, out_size );
fprintf( stderr, "lzcheck: Sync error at pos %d in_size = %d, out_size = %d, leading garbage = %d\n",
l, in_size, out_size, leading_garbage );
for( i = 0; i < in_size; ++i )
fputc( in_buffer[l+i], stderr );
if( in_buffer[l+in_size-1] != '\n' )
@ -231,7 +231,7 @@ int main( const int argc, const char * const argv[] )
}
/* fprintf( stderr, "lzcheck: Testing file '%s'\n", argv[1] ); */
retval = lzcheck( file, 65535 );
retval = lzcheck( file, 65535 ); /* 65535,16 chooses fast encoder */
if( retval == 0 )
{ rewind( file ); retval = lzcheck( file, 1 << 20 ); }
fclose( file );

31
lzip.h
View file

@ -1,5 +1,5 @@
/* Lzlib - Compression library for the lzip format
Copyright (C) 2009-2015 Antonio Diaz Diaz.
Copyright (C) 2009-2016 Antonio Diaz Diaz.
This library is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -60,6 +60,7 @@ enum {
max_dictionary_bits = 29,
max_dictionary_size = 1 << max_dictionary_bits,
literal_context_bits = 3,
literal_pos_state_bits = 0, /* not used */
pos_state_bits = 2,
pos_states = 1 << pos_state_bits,
pos_state_mask = pos_states - 1,
@ -100,8 +101,8 @@ typedef int Bit_model;
static inline void Bm_init( Bit_model * const probability )
{ *probability = bit_model_total / 2; }
static inline void Bm_array_init( Bit_model * const p, const int size )
{ int i = 0; while( i < size ) p[i++] = bit_model_total / 2; }
static inline void Bm_array_init( Bit_model bm[], const int size )
{ int i; for( i = 0; i < size; ++i ) Bm_init( &bm[i] ); }
struct Len_model
{
@ -183,6 +184,11 @@ static inline void CRC32_update_buf( uint32_t * const crc,
}
static inline bool isvalid_ds( const unsigned dictionary_size )
{ return ( dictionary_size >= min_dictionary_size &&
dictionary_size <= max_dictionary_size ); }
static inline int real_bits( unsigned value )
{
int bits = 0;
@ -204,6 +210,14 @@ static inline void Fh_set_magic( File_header data )
static inline bool Fh_verify_magic( const File_header data )
{ return ( memcmp( data, magic_string, 4 ) == 0 ); }
/* detect truncated header */
static inline bool Fh_verify_prefix( const File_header data, const int size )
{
int i; for( i = 0; i < size && i < 4; ++i )
if( data[i] != magic_string[i] ) return false;
return ( size > 0 );
}
static inline uint8_t Fh_version( const File_header data )
{ return data[4]; }
@ -220,8 +234,7 @@ static inline unsigned Fh_get_dictionary_size( const File_header data )
static inline bool Fh_set_dictionary_size( File_header data, const unsigned sz )
{
if( sz >= min_dictionary_size && sz <= max_dictionary_size )
{
if( !isvalid_ds( sz ) ) return false;
data[5] = real_bits( sz - 1 );
if( sz > min_dictionary_size )
{
@ -234,17 +247,11 @@ static inline bool Fh_set_dictionary_size( File_header data, const unsigned sz )
}
return true;
}
return false;
}
static inline bool Fh_verify( const File_header data )
{
if( Fh_verify_magic( data ) && Fh_verify_version( data ) )
{
const unsigned dictionary_size = Fh_get_dictionary_size( data );
return ( dictionary_size >= min_dictionary_size &&
dictionary_size <= max_dictionary_size );
}
return isvalid_ds( Fh_get_dictionary_size( data ) );
return false;
}

74
lzlib.c
View file

@ -1,5 +1,5 @@
/* Lzlib - Compression library for the lzip format
Copyright (C) 2009-2015 Antonio Diaz Diaz.
Copyright (C) 2009-2016 Antonio Diaz Diaz.
This library is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -75,6 +75,7 @@ struct LZ_Decoder
enum LZ_Errno lz_errno;
File_header member_header; /* header of current member */
bool fatal;
bool first_header; /* true until first header is read */
bool seeking;
};
@ -88,6 +89,7 @@ static void LZ_Decoder_init( struct LZ_Decoder * const d )
d->lz_errno = LZ_ok;
for( i = 0; i < Fh_size; ++i ) d->member_header[i] = 0;
d->fatal = false;
d->first_header = true;
d->seeking = false;
}
@ -201,7 +203,7 @@ int LZ_compress_finish( struct LZ_Encoder * const e )
if( Mb_data_position( &e->lz_encoder_base->mb ) == 0 &&
LZ_compress_total_out_size( e ) == Fh_size )
{
Mb_adjust_distionary_size( &e->lz_encoder_base->mb );
Mb_adjust_dictionary_size( &e->lz_encoder_base->mb );
Fh_set_dictionary_size( e->lz_encoder_base->renc.header,
e->lz_encoder_base->mb.dictionary_size );
e->lz_encoder_base->renc.cb.buffer[5] = e->lz_encoder_base->renc.header[5];
@ -243,6 +245,7 @@ int LZ_compress_read( struct LZ_Encoder * const e,
{
int out_size = 0;
if( !verify_encoder( e ) || e->fatal ) return -1;
if( size < 0 ) return 0;
do {
if( ( e->flz_encoder && !FLZe_encode_member( e->flz_encoder ) ) ||
( e->lz_encoder && !LZe_encode_member( e->lz_encoder ) ) )
@ -359,7 +362,8 @@ int LZ_decompress_close( struct LZ_Decoder * const d )
int LZ_decompress_finish( struct LZ_Decoder * const d )
{
if( !verify_decoder( d ) || d->fatal ) return -1;
if( d->seeking ) { d->seeking = false; Rd_purge( d->rdec ); }
if( d->seeking )
{ d->seeking = false; d->partial_in_size += Rd_purge( d->rdec ); }
else Rd_finish( d->rdec );
return 0;
}
@ -375,6 +379,7 @@ int LZ_decompress_reset( struct LZ_Decoder * const d )
Rd_reset( d->rdec );
d->lz_errno = LZ_ok;
d->fatal = false;
d->first_header = true;
d->seeking = false;
return 0;
}
@ -382,15 +387,17 @@ int LZ_decompress_reset( struct LZ_Decoder * const d )
int LZ_decompress_sync_to_member( struct LZ_Decoder * const d )
{
int skipped = 0;
if( !verify_decoder( d ) ) return -1;
if( d->lz_decoder )
{ LZd_free( d->lz_decoder ); free( d->lz_decoder ); d->lz_decoder = 0; }
if( Rd_find_header( d->rdec ) ) d->seeking = false;
if( Rd_find_header( d->rdec, &skipped ) ) d->seeking = false;
else
{
if( !d->rdec->at_stream_end ) d->seeking = true;
else { d->seeking = false; Rd_purge( d->rdec ); }
else { d->seeking = false; d->partial_in_size += Rd_purge( d->rdec ); }
}
d->partial_in_size += skipped;
d->lz_errno = LZ_ok;
d->fatal = false;
return 0;
@ -402,32 +409,49 @@ int LZ_decompress_read( struct LZ_Decoder * const d,
{
int result;
if( !verify_decoder( d ) || d->fatal ) return -1;
if( d->seeking ) return 0;
if( d->seeking || size < 0 ) return 0;
if( d->lz_decoder && LZd_member_finished( d->lz_decoder ) )
{
d->partial_in_size += d->rdec->member_position;
d->partial_out_size += LZd_data_position( d->lz_decoder );
LZd_free( d->lz_decoder ); free( d->lz_decoder ); d->lz_decoder = 0;
}
if( !d->lz_decoder )
{
if( Rd_available_bytes( d->rdec ) < 5 + Fh_size )
int rd;
d->partial_in_size += d->rdec->member_position;
d->rdec->member_position = 0;
if( Rd_available_bytes( d->rdec ) < Fh_size + 5 &&
!d->rdec->at_stream_end ) return 0;
if( Rd_finished( d->rdec ) && !d->first_header ) return 0;
rd = Rd_read_data( d->rdec, d->member_header, Fh_size );
if( Rd_finished( d->rdec ) )
{
if( !d->rdec->at_stream_end || Rd_finished( d->rdec ) ) return 0;
/* set position at EOF */
d->partial_in_size += Rd_available_bytes( d->rdec );
if( Rd_available_bytes( d->rdec ) <= Fh_size ||
!Rd_read_header( d->rdec, d->member_header ) )
d->lz_errno = LZ_header_error;
else
if( rd <= 0 || Fh_verify_prefix( d->member_header, rd ) )
d->lz_errno = LZ_unexpected_eof;
else
d->lz_errno = LZ_header_error;
d->fatal = true;
return -1;
}
if( !Rd_read_header( d->rdec, d->member_header ) )
if( !Fh_verify( d->member_header ) )
{
/* unreading the header prevents sync_to_member from skipping a member
if leading garbage is shorter than a full header; "lgLZIP\x01\x0C" */
if( Rd_unread_data( d->rdec, rd ) )
d->lz_errno = LZ_header_error;
else
d->lz_errno = LZ_library_error;
d->fatal = true;
return -1;
}
d->first_header = false;
if( Rd_available_bytes( d->rdec ) < 5 )
{
/* set position at EOF */
d->rdec->member_position += Cb_used_bytes( &d->rdec->cb );
Cb_reset( &d->rdec->cb );
d->lz_errno = LZ_unexpected_eof;
d->fatal = true;
return -1;
}
@ -441,11 +465,16 @@ int LZ_decompress_read( struct LZ_Decoder * const d,
d->fatal = true;
return -1;
}
d->rdec->reload_pending = true;
}
result = LZd_decode_member( d->lz_decoder );
if( result != 0 )
{
if( result == 2 ) d->lz_errno = LZ_unexpected_eof;
if( result == 2 )
{ d->lz_errno = LZ_unexpected_eof;
d->rdec->member_position += Cb_used_bytes( &d->rdec->cb );
Cb_reset( &d->rdec->cb ); }
else if( result == 5 ) d->lz_errno = LZ_library_error;
else d->lz_errno = LZ_data_error;
d->fatal = true;
return -1;
@ -459,12 +488,14 @@ int LZ_decompress_write( struct LZ_Decoder * const d,
{
int result;
if( !verify_decoder( d ) || d->fatal ) return -1;
if( size < 0 ) return 0;
result = Rd_write_data( d->rdec, buffer, size );
while( d->seeking )
{
int size2;
if( Rd_find_header( d->rdec ) ) d->seeking = false;
int size2, skipped = 0;
if( Rd_find_header( d->rdec, &skipped ) ) d->seeking = false;
d->partial_in_size += skipped;
if( result >= size ) break;
size2 = Rd_write_data( d->rdec, buffer + result, size - result );
if( size2 > 0 ) result += size2;
@ -535,18 +566,15 @@ unsigned long long LZ_decompress_data_position( struct LZ_Decoder * const d )
unsigned long long LZ_decompress_member_position( struct LZ_Decoder * const d )
{
if( verify_decoder( d ) && d->lz_decoder )
if( !verify_decoder( d ) ) return 0;
return d->rdec->member_position;
return 0;
}
unsigned long long LZ_decompress_total_in_size( struct LZ_Decoder * const d )
{
if( !verify_decoder( d ) ) return 0;
if( d->lz_decoder )
return d->partial_in_size + d->rdec->member_position;
return d->partial_in_size;
}

6
lzlib.h
View file

@ -1,5 +1,5 @@
/* Lzlib - Compression library for the lzip format
Copyright (C) 2009-2015 Antonio Diaz Diaz.
Copyright (C) 2009-2016 Antonio Diaz Diaz.
This library is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -29,7 +29,9 @@
extern "C" {
#endif
static const char * const LZ_version_string = "1.7";
#define LZ_API_VERSION 1
static const char * const LZ_version_string = "1.8";
enum LZ_Errno { LZ_ok = 0, LZ_bad_argument, LZ_mem_error,
LZ_sequence_error, LZ_header_error, LZ_unexpected_eof,

139
main.c
View file

@ -1,5 +1,5 @@
/* Minilzip - Test program for the lzlib library
Copyright (C) 2009-2015 Antonio Diaz Diaz.
Copyright (C) 2009-2016 Antonio Diaz Diaz.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -73,10 +73,11 @@ void cleanup_and_fail( const int retval );
void show_error( const char * const msg, const int errcode, const bool help );
void internal_error( const char * const msg );
int verbosity = 0;
const char * const Program_name = "Minilzip";
const char * const program_name = "minilzip";
const char * const program_year = "2015";
const char * const program_year = "2016";
const char * invocation_name = 0;
struct { const char * from; const char * to; } const known_extensions[] = {
@ -94,10 +95,6 @@ enum Mode { m_compress, m_decompress, m_test };
char * output_filename = 0;
int outfd = -1;
int verbosity = 0;
const mode_t usr_rw = S_IRUSR | S_IWUSR;
const mode_t all_rw = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH;
mode_t outfd_mode = S_IRUSR | S_IWUSR;
bool delete_output_on_interrupt = false;
@ -110,7 +107,8 @@ struct Pretty_print
};
static void Pp_init( struct Pretty_print * const pp,
const char * const filenames[], const int num_filenames )
const char * const filenames[],
const int num_filenames, const int verbosity )
{
unsigned stdin_name_len;
int i;
@ -120,6 +118,7 @@ static void Pp_init( struct Pretty_print * const pp,
pp->first_post = false;
stdin_name_len = strlen( pp->stdin_name );
if( verbosity <= 0 ) return;
for( i = 0; i < num_filenames; ++i )
{
const char * const s = filenames[i];
@ -147,10 +146,11 @@ static void Pp_show_msg( struct Pretty_print * const pp, const char * const msg
{
if( pp->first_post )
{
int i, len = pp->longest_name - strlen( pp->name );
unsigned i;
pp->first_post = false;
fprintf( stderr, " %s: ", pp->name );
for( i = 0; i < len; ++i ) fputc( ' ', stderr );
for( i = strlen( pp->name ); i < pp->longest_name; ++i )
fputc( ' ', stderr );
if( !msg ) fflush( stderr );
}
if( msg ) fprintf( stderr, "%s\n", msg );
@ -165,14 +165,15 @@ static void show_help( void )
printf( "\nOptions:\n"
" -h, --help display this help and exit\n"
" -V, --version output version information and exit\n"
" -a, --trailing-error exit with error status if trailing data\n"
" -b, --member-size=<bytes> set member size limit in bytes\n"
" -c, --stdout send output to standard output\n"
" -c, --stdout write to standard output, keep input files\n"
" -d, --decompress decompress\n"
" -f, --force overwrite existing output files\n"
" -F, --recompress force re-compression of compressed files\n"
" -k, --keep keep (don't delete) input files\n"
" -m, --match-length=<bytes> set match length limit in bytes [36]\n"
" -o, --output=<file> if reading stdin, place the output into <file>\n"
" -o, --output=<file> if reading standard input, write to <file>\n"
" -q, --quiet suppress all messages\n"
" -s, --dictionary-size=<bytes> set dictionary size limit in bytes [8 MiB]\n"
" -S, --volume-size=<bytes> set volume size limit in bytes\n"
@ -181,13 +182,15 @@ static void show_help( void )
" -0 .. -9 set compression level [default 6]\n"
" --fast alias for -0\n"
" --best alias for -9\n"
"If no file names are given, minilzip compresses or decompresses\n"
"from standard input to standard output.\n"
"If no file names are given, or if a file is '-', minilzip compresses or\n"
"decompresses from standard input to standard output.\n"
"Numbers may be followed by a multiplier: k = kB = 10^3 = 1000,\n"
"Ki = KiB = 2^10 = 1024, M = 10^6, Mi = 2^20, G = 10^9, Gi = 2^30, etc...\n"
"The bidimensional parameter space of LZMA can't be mapped to a linear\n"
"Dictionary sizes 12 to 29 are interpreted as powers of two, meaning 2^12\n"
"to 2^29 bytes.\n"
"\nThe bidimensional parameter space of LZMA can't be mapped to a linear\n"
"scale optimal for all files. If your files are large, very repetitive,\n"
"etc, you may need to use the --match-length and --dictionary-size\n"
"etc, you may need to use the --dictionary-size and --match-length\n"
"options directly to achieve optimal performance.\n"
"\nExit status: 0 for a normal exit, 1 for environmental problems (file\n"
"not found, invalid flags, I/O errors, etc), 2 to indicate a corrupt or\n"
@ -246,11 +249,10 @@ static unsigned long long getnum( const char * const ptr,
if( !errno && tail[0] )
{
const int factor = ( tail[1] == 'i' ) ? 1024 : 1000;
int exponent = 0, i;
bool bad_multiplier = false;
int exponent = 0; /* 0 = bad multiplier */
int i;
switch( tail[0] )
{
case ' ': break;
case 'Y': exponent = 8; break;
case 'Z': exponent = 7; break;
case 'E': exponent = 6; break;
@ -258,13 +260,10 @@ static unsigned long long getnum( const char * const ptr,
case 'T': exponent = 4; break;
case 'G': exponent = 3; break;
case 'M': exponent = 2; break;
case 'K': if( factor == 1024 ) exponent = 1; else bad_multiplier = true;
break;
case 'k': if( factor == 1000 ) exponent = 1; else bad_multiplier = true;
break;
default : bad_multiplier = true;
case 'K': if( factor == 1024 ) exponent = 1; break;
case 'k': if( factor == 1000 ) exponent = 1; break;
}
if( bad_multiplier )
if( exponent <= 0 )
{
show_error( "Bad multiplier in numerical argument.", 0, true );
exit( 1 );
@ -343,7 +342,7 @@ static int open_instream( const char * const name, struct stat * const in_statsp
const bool can_read = ( i == 0 &&
( S_ISBLK( mode ) || S_ISCHR( mode ) ||
S_ISFIFO( mode ) || S_ISSOCK( mode ) ) );
const bool no_ofile = to_stdout || program_mode == m_test;
const bool no_ofile = ( to_stdout || program_mode == m_test );
if( i != 0 || ( !S_ISREG( mode ) && ( !can_read || !no_ofile ) ) )
{
if( verbosity >= 0 )
@ -409,13 +408,17 @@ static void set_d_outname( const char * const name, const int i )
}
static bool open_outstream( const bool force )
static bool open_outstream( const bool force, const bool from_stdin )
{
const mode_t usr_rw = S_IRUSR | S_IWUSR;
const mode_t all_rw = usr_rw | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH;
const mode_t outfd_mode = from_stdin ? all_rw : usr_rw;
int flags = O_CREAT | O_WRONLY | O_BINARY;
if( force ) flags |= O_TRUNC; else flags |= O_EXCL;
outfd = open( output_filename, flags, outfd_mode );
if( outfd < 0 && verbosity >= 0 )
if( outfd >= 0 ) delete_output_on_interrupt = true;
else if( verbosity >= 0 )
{
if( errno == EEXIST )
fprintf( stderr, "%s: Output file '%s' already exists, skipping.\n",
@ -476,7 +479,11 @@ static void close_and_set_permissions( const struct stat * const in_statsp )
fchmod( outfd, mode & ~( S_ISUID | S_ISGID | S_ISVTX ) ) != 0 )
warning = true;
}
if( close( outfd ) != 0 ) cleanup_and_fail( 1 );
if( close( outfd ) != 0 )
{
show_error( "Error closing output file", errno, false );
cleanup_and_fail( 1 );
}
outfd = -1;
delete_output_on_interrupt = false;
if( in_statsp )
@ -552,8 +559,8 @@ static int do_compress( struct LZ_Encoder * const encoder,
unsigned long long partial_volume_size = 0;
enum { buffer_size = 65536 };
uint8_t buffer[buffer_size];
if( verbosity >= 1 ) Pp_show_msg( pp, 0 );
while( true )
{
int in_size = 0, out_size;
@ -606,8 +613,7 @@ static int do_compress( struct LZ_Encoder * const encoder,
close_and_set_permissions( in_statsp );
if( !next_filename() )
{ Pp_show_msg( pp, "Too many volume files." ); return 1; }
if( !open_outstream( true ) ) return 1;
delete_output_on_interrupt = true;
if( !open_outstream( true, !in_statsp ) ) return 1;
}
}
size = min( member_size, volume_size - partial_volume_size );
@ -671,7 +677,8 @@ static int compress( const unsigned long long member_size,
static int do_decompress( struct LZ_Decoder * const decoder, const int infd,
struct Pretty_print * const pp, const bool testing )
struct Pretty_print * const pp,
const bool ignore_trailing, const bool testing )
{
enum { buffer_size = 65536 };
uint8_t buffer[buffer_size];
@ -679,11 +686,10 @@ static int do_decompress( struct LZ_Decoder * const decoder, const int infd,
for( first_member = true; ; )
{
int in_size = min( LZ_decompress_write_size( decoder ), buffer_size );
int out_size = 0;
if( in_size > 0 )
const int max_in_size = min( LZ_decompress_write_size( decoder ), buffer_size );
int in_size = 0, out_size = 0;
if( max_in_size > 0 )
{
const int max_in_size = in_size;
in_size = readblock( infd, buffer, max_in_size );
if( in_size != max_in_size && errno )
{
@ -715,19 +721,18 @@ static int do_decompress( struct LZ_Decoder * const decoder, const int infd,
{
if( verbosity >= 1 )
{
const unsigned long long data_position = LZ_decompress_data_position( decoder );
const unsigned long long data_size = LZ_decompress_data_position( decoder );
const unsigned long long member_size = LZ_decompress_member_position( decoder );
Pp_show_msg( pp, 0 );
show_header( LZ_decompress_dictionary_size( decoder ) );
if( verbosity >= 2 && data_position > 0 && member_size > 0 )
if( verbosity >= 2 && data_size > 0 && member_size > 0 )
fprintf( stderr, "%6.3f:1, %6.3f bits/byte, %5.2f%% saved. ",
(double)data_position / member_size,
( 8.0 * member_size ) / data_position,
100.0 * ( 1.0 - ( (double)member_size / data_position ) ) );
(double)data_size / member_size,
( 8.0 * member_size ) / data_size,
100.0 * ( 1.0 - ( (double)member_size / data_size ) ) );
if( verbosity >= 4 )
fprintf( stderr, "data CRC %08X, data size %9llu, member size %8llu. ",
LZ_decompress_data_crc( decoder ),
data_position, member_size );
LZ_decompress_data_crc( decoder ), data_size, member_size );
fputs( testing ? "ok\n" : "done\n", stderr );
}
first_member = false; Pp_reset( pp );
@ -737,11 +742,18 @@ static int do_decompress( struct LZ_Decoder * const decoder, const int infd,
if( out_size < 0 || ( first_member && out_size == 0 ) )
{
const enum LZ_Errno lz_errno = LZ_decompress_errno( decoder );
if( lz_errno == LZ_header_error || ( first_member && out_size == 0 ) )
if( lz_errno == LZ_unexpected_eof &&
LZ_decompress_member_position( decoder ) <= 6 )
{ Pp_show_msg( pp, "File ends unexpectedly at member header." );
return 2; }
if( lz_errno == LZ_header_error )
{
if( !first_member ) break; /* trailing garbage */
Pp_show_msg( pp, "Bad magic number (file not in lzip format)." );
return 2;
if( first_member )
{ Pp_show_msg( pp, "Bad magic number (file not in lzip format)." );
return 2; }
else if( !ignore_trailing )
{ show_error( "Trailing data not allowed.", 0, false ); return 2; }
break;
}
if( lz_errno == LZ_mem_error )
{ Pp_show_msg( pp, "Not enough memory." ); return 1; }
@ -767,14 +779,14 @@ static int do_decompress( struct LZ_Decoder * const decoder, const int infd,
static int decompress( const int infd, struct Pretty_print * const pp,
const bool testing )
const bool ignore_trailing, const bool testing )
{
struct LZ_Decoder * const decoder = LZ_decompress_open();
int retval;
if( !decoder || LZ_decompress_errno( decoder ) != LZ_ok )
{ Pp_show_msg( pp, "Not enough memory." ); retval = 1; }
else retval = do_decompress( decoder, infd, pp, testing );
else retval = do_decompress( decoder, infd, pp, ignore_trailing, testing );
LZ_decompress_close( decoder );
return retval;
@ -799,8 +811,7 @@ static void set_signals( void )
void show_error( const char * const msg, const int errcode, const bool help )
{
if( verbosity >= 0 )
{
if( verbosity < 0 ) return;
if( msg && msg[0] )
{
fprintf( stderr, "%s: %s", program_name, msg );
@ -811,7 +822,6 @@ void show_error( const char * const msg, const int errcode, const bool help )
fprintf( stderr, "Try '%s --help' for more information.\n",
invocation_name );
}
}
void internal_error( const char * const msg )
@ -828,7 +838,7 @@ int main( const int argc, const char * const argv[] )
to the corresponding LZMA compression modes. */
const struct Lzma_options option_mapping[] =
{
{ 65535, 16 }, /* -0 */
{ 65535, 16 }, /* -0 (65535,16 chooses fast encoder) */
{ 1 << 20, 5 }, /* -1 */
{ 3 << 19, 6 }, /* -2 */
{ 1 << 21, 8 }, /* -3 */
@ -854,7 +864,9 @@ int main( const int argc, const char * const argv[] )
int i;
bool filenames_given = false;
bool force = false;
bool ignore_trailing = true;
bool keep_input_files = false;
bool stdin_used = false;
bool recompress = false;
bool to_stdout = false;
struct Pretty_print pp;
@ -871,6 +883,7 @@ int main( const int argc, const char * const argv[] )
{ '7', 0, ap_no },
{ '8', 0, ap_no },
{ '9', "best", ap_no },
{ 'a', "trailing-error", ap_no },
{ 'b', "member-size", ap_yes },
{ 'c', "stdout", ap_no },
{ 'd', "decompress", ap_no },
@ -913,6 +926,7 @@ int main( const int argc, const char * const argv[] )
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9':
encoder_options = option_mapping[code-'0']; break;
case 'a': ignore_trailing = false; break;
case 'b': member_size = getnum( arg, 100000, max_member_size ); break;
case 'c': to_stdout = true; break;
case 'd': program_mode = m_decompress; break;
@ -958,7 +972,7 @@ int main( const int argc, const char * const argv[] )
( filenames_given || default_output_filename[0] ) )
set_signals();
Pp_init( &pp, filenames, num_filenames );
Pp_init( &pp, filenames, num_filenames, verbosity );
output_filename = resize_buffer( output_filename, 1 );
for( i = 0; i < num_filenames; ++i )
@ -970,6 +984,7 @@ int main( const int argc, const char * const argv[] )
if( !filenames[i][0] || strcmp( filenames[i], "-" ) == 0 )
{
if( stdin_used ) continue; else stdin_used = true;
input_filename = "";
infd = STDIN_FILENO;
if( program_mode != m_test )
@ -986,8 +1001,7 @@ int main( const int argc, const char * const argv[] )
strlen( default_output_filename ) + 1 );
strcpy( output_filename, default_output_filename );
}
outfd_mode = all_rw;
if( !open_outstream( force ) )
if( !open_outstream( force, true ) )
{
if( retval < 1 ) retval = 1;
close( infd ); infd = -1;
@ -1011,8 +1025,7 @@ int main( const int argc, const char * const argv[] )
if( program_mode == m_compress )
set_c_outname( input_filename, volume_size > 0 );
else set_d_outname( input_filename, eindex );
outfd_mode = usr_rw;
if( !open_outstream( force ) )
if( !open_outstream( force, false ) )
{
if( retval < 1 ) retval = 1;
close( infd ); infd = -1;
@ -1022,17 +1035,19 @@ int main( const int argc, const char * const argv[] )
}
}
if( !check_tty( infd, program_mode ) ) return 1;
if( !check_tty( infd, program_mode ) )
{
if( retval < 1 ) retval = 1;
cleanup_and_fail( retval );
}
if( output_filename[0] && !to_stdout && program_mode != m_test )
delete_output_on_interrupt = true;
in_statsp = input_filename[0] ? &in_stats : 0;
Pp_set_name( &pp, input_filename );
if( program_mode == m_compress )
tmp = compress( member_size, volume_size, infd, &encoder_options, &pp,
in_statsp );
else
tmp = decompress( infd, &pp, program_mode == m_test );
tmp = decompress( infd, &pp, ignore_trailing, program_mode == m_test );
if( tmp > retval ) retval = tmp;
if( tmp && program_mode != m_test ) cleanup_and_fail( retval );

111
testsuite/check.sh
View file

@ -1,6 +1,6 @@
#! /bin/sh
# check script for Lzlib - A compression library for lzip files
# Copyright (C) 2009-2015 Antonio Diaz Diaz.
# Copyright (C) 2009-2016 Antonio Diaz Diaz.
#
# This script is free software: you have unlimited permission
# to copy, distribute and modify it.
@ -19,35 +19,40 @@ if [ ! -f "${LZIP}" ] || [ ! -x "${LZIP}" ] ; then
exit 1
fi
if [ -e "${LZIP}" ] 2> /dev/null ; then true
else
echo "$0: a POSIX shell is required to run the tests"
echo "Try bash -c \"$0 $1 $2\""
exit 1
fi
if [ -d tmp ] ; then rm -rf tmp ; fi
mkdir tmp
cd "${objdir}"/tmp
cd "${objdir}"/tmp || framework_failure
cat "${testdir}"/test.txt > in || framework_failure
in_lz="${testdir}"/test.txt.lz
test2="${testdir}"/test2.txt
fail=0
printf "testing lzlib-%s..." "$2"
"${LZIP}" -cqm4 in > /dev/null
if [ $? = 1 ] ; then printf . ; else printf - ; fail=1 ; fi
"${LZIP}" -cqm274 in > /dev/null
if [ $? = 1 ] ; then printf . ; else printf - ; fail=1 ; fi
"${LZIP}" -cqs-1 in > /dev/null
if [ $? = 1 ] ; then printf . ; else printf - ; fail=1 ; fi
"${LZIP}" -cqs0 in > /dev/null
if [ $? = 1 ] ; then printf . ; else printf - ; fail=1 ; fi
"${LZIP}" -cqs4095 in > /dev/null
if [ $? = 1 ] ; then printf . ; else printf - ; fail=1 ; fi
"${LZIP}" -cqs513MiB in > /dev/null
if [ $? = 1 ] ; then printf . ; else printf - ; fail=1 ; fi
printf " in: Bad magic number (file not in lzip format).\n" > msg
"${LZIP}" -t in 2> out
if [ $? = 2 ] && cmp out msg ; then printf . ; else printf - ; fail=1 ; fi
printf " (stdin): Bad magic number (file not in lzip format).\n" > msg
"${LZIP}" -t < in 2> out
if [ $? = 2 ] && cmp out msg ; then printf . ; else printf - ; fail=1 ; fi
rm -f out msg
"${LZIP}" -fkqm4 in
if [ $? = 1 ] && [ ! -e in.lz ] ; then printf . ; else printf - ; fail=1 ; fi
"${LZIP}" -fkqm274 in
if [ $? = 1 ] && [ ! -e in.lz ] ; then printf . ; else printf - ; fail=1 ; fi
"${LZIP}" -fkqs-1 in
if [ $? = 1 ] && [ ! -e in.lz ] ; then printf . ; else printf - ; fail=1 ; fi
"${LZIP}" -fkqs0 in
if [ $? = 1 ] && [ ! -e in.lz ] ; then printf . ; else printf - ; fail=1 ; fi
"${LZIP}" -fkqs4095 in
if [ $? = 1 ] && [ ! -e in.lz ] ; then printf . ; else printf - ; fail=1 ; fi
"${LZIP}" -fkqs513MiB in
if [ $? = 1 ] && [ ! -e in.lz ] ; then printf . ; else printf - ; fail=1 ; fi
"${LZIP}" -tq in
if [ $? = 2 ] ; then printf . ; else printf - ; fail=1 ; fi
"${LZIP}" -tq < in
if [ $? = 2 ] ; then printf . ; else printf - ; fail=1 ; fi
"${LZIP}" -cdq in
if [ $? = 2 ] ; then printf . ; else printf - ; fail=1 ; fi
"${LZIP}" -cdq < in
@ -57,31 +62,59 @@ if [ $? = 2 ] ; then printf . ; else printf - ; fail=1 ; fi
dd if="${in_lz}" bs=1 count=20 2> /dev/null | "${LZIP}" -tq
if [ $? = 2 ] ; then printf . ; else printf - ; fail=1 ; fi
"${LZIP}" -t "${in_lz}" || fail=1
printf "\ntesting decompression..."
"${LZIP}" -t "${in_lz}"
if [ $? = 0 ] ; then printf . ; else printf - ; fail=1 ; fi
"${LZIP}" -cd "${in_lz}" > copy || fail=1
cmp in copy || fail=1
printf .
"${LZIP}" -t "${testdir}"/test_sync.lz || fail=1
"${LZIP}" -t "${testdir}"/test_sync.lz
if [ $? = 0 ] ; then printf . ; else printf - ; fail=1 ; fi
"${LZIP}" -cd "${testdir}"/test_sync.lz > copy || fail=1
cmp in copy || fail=1
printf .
rm -f copy
cat "${in_lz}" > copy.lz || framework_failure
printf "to be overwritten" > copy || framework_failure
"${LZIP}" -df copy.lz || fail=1
"${LZIP}" -dk copy.lz || fail=1
cmp in copy || fail=1
printf .
printf "to be overwritten" > copy || framework_failure
"${LZIP}" -dq copy.lz
if [ $? = 1 ] ; then printf . ; else printf - ; fail=1 ; fi
"${LZIP}" -df copy.lz
if [ $? = 0 ] && [ ! -e copy.lz ] && cmp in copy ; then
printf . ; else printf - ; fail=1 ; fi
printf "to be overwritten" > copy || framework_failure
"${LZIP}" -df -o copy < "${in_lz}" || fail=1
cmp in copy || fail=1
printf .
rm -f copy
"${LZIP}" -s16 < in > anyothername || fail=1
"${LZIP}" -d anyothername || fail=1
cmp in anyothername.out || fail=1
printf .
"${LZIP}" -d -o copy - anyothername - < "${in_lz}"
if [ $? = 0 ] && cmp in copy && cmp in anyothername.out ; then
printf . ; else printf - ; fail=1 ; fi
rm -f copy anyothername.out
"${LZIP}" -tq in "${in_lz}"
if [ $? = 2 ] ; then printf . ; else printf - ; fail=1 ; fi
"${LZIP}" -tq foo.lz "${in_lz}"
if [ $? = 1 ] ; then printf . ; else printf - ; fail=1 ; fi
"${LZIP}" -cdq in "${in_lz}" > copy
if [ $? = 2 ] && cat copy in | cmp in - ; then printf . ; else printf - ; fail=1 ; fi
"${LZIP}" -cdq foo.lz "${in_lz}" > copy
if [ $? = 1 ] && cmp in copy ; then printf . ; else printf - ; fail=1 ; fi
rm -f copy
cat "${in_lz}" > copy.lz || framework_failure
"${LZIP}" -dq in copy.lz
if [ $? = 2 ] && [ -e copy.lz ] && [ ! -e copy ] && [ ! -e in.out ] ; then
printf . ; else printf - ; fail=1 ; fi
"${LZIP}" -dq foo.lz copy.lz
if [ $? = 1 ] && [ ! -e copy.lz ] && [ ! -e foo ] && cmp in copy ; then
printf . ; else printf - ; fail=1 ; fi
cat in in > in2 || framework_failure
"${LZIP}" -s16 -o copy2 < in2 || fail=1
@ -91,12 +124,23 @@ cmp in2 copy2 || fail=1
printf .
printf "garbage" >> copy2.lz || framework_failure
rm -f copy2
"${LZIP}" -atq copy2.lz
if [ $? = 2 ] ; then printf . ; else printf - ; fail=1 ; fi
"${LZIP}" -atq < copy2.lz
if [ $? = 2 ] ; then printf . ; else printf - ; fail=1 ; fi
"${LZIP}" -adkq copy2.lz
if [ $? = 2 ] && [ ! -e copy2 ] ; then printf . ; else printf - ; fail=1 ; fi
"${LZIP}" -adkq -o copy2 < copy2.lz
if [ $? = 2 ] && [ ! -e copy2 ] ; then printf . ; else printf - ; fail=1 ; fi
printf "to be overwritten" > copy2 || framework_failure
"${LZIP}" -df copy2.lz || fail=1
cmp in2 copy2 || fail=1
printf .
"${LZIP}" -cfq "${in_lz}" > out
printf "\ntesting compression..."
"${LZIP}" -cfq "${in_lz}" > out # /dev/null is a tty on OS/2
if [ $? = 1 ] ; then printf . ; else printf - ; fail=1 ; fi
"${LZIP}" -cF -s16 "${in_lz}" > out || fail=1
"${LZIP}" -cd out | "${LZIP}" -d > copy || fail=1
@ -135,11 +179,18 @@ done
printf .
"${BBEXAMPLE}" in || fail=1
printf .
"${BBEXAMPLE}" out || fail=1
printf .
"${BBEXAMPLE}" ${test2} || fail=1
printf .
"${LZCHECK}" in || fail=1
printf .
"${LZCHECK}" out || fail=1
printf .
"${LZCHECK}" ${test2} || fail=1
printf .
echo
if [ ${fail} = 0 ] ; then

9
testsuite/test2.txt Normal file
View file

@ -0,0 +1,9 @@
The
quick
brown
fox
jumps
over
the
lazy
dog.