Merging upstream version 1.3.
Signed-off-by: Daniel Baumann <daniel@debian.org>
This commit is contained in:
parent
14c6cd47d9
commit
ef2fe9ecc0
GPG key ID:
FBB4F0E80A80222F
18 changed files with 733 additions and 665 deletions
361
encoder.c
361
encoder.c
|
|
@ -1,5 +1,5 @@
|
|||
/* Clzip - Data compressor based on the LZMA algorithm
|
||||
Copyright (C) 2010, 2011 Antonio Diaz Diaz.
|
||||
Copyright (C) 2010, 2011, 2012 Antonio Diaz Diaz.
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
|
|
@ -19,9 +19,9 @@
|
|||
|
||||
#include <errno.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include "clzip.h"
|
||||
#include "encoder.h"
|
||||
|
|
@ -40,65 +40,75 @@ bool Mf_read_block( struct Matchfinder * const mf )
|
|||
mf->stream_pos += rd;
|
||||
if( rd != size && errno )
|
||||
{ show_error( "Read error", errno, false ); cleanup_and_fail( 1 ); }
|
||||
mf->at_stream_end = ( rd < size );
|
||||
if( rd < size )
|
||||
{ mf->at_stream_end = true; mf->pos_limit = mf->buffer_size; }
|
||||
}
|
||||
return mf->pos < mf->stream_pos;
|
||||
}
|
||||
|
||||
|
||||
void Mf_init( struct Matchfinder * const mf,
|
||||
void Mf_normalize_pos( struct Matchfinder * const mf )
|
||||
{
|
||||
if( mf->pos > mf->stream_pos )
|
||||
internal_error( "pos > stream_pos in Mf_normalize_pos" );
|
||||
if( !mf->at_stream_end )
|
||||
{
|
||||
int i;
|
||||
const int offset = mf->pos - mf->dictionary_size - before_size;
|
||||
const int size = mf->stream_pos - offset;
|
||||
memmove( mf->buffer, mf->buffer + offset, size );
|
||||
mf->partial_data_pos += offset;
|
||||
mf->pos -= offset;
|
||||
mf->stream_pos -= offset;
|
||||
for( i = 0; i < num_prev_positions; ++i )
|
||||
if( mf->prev_positions[i] >= 0 ) mf->prev_positions[i] -= offset;
|
||||
for( i = 0; i < 2 * mf->dictionary_size; ++i )
|
||||
if( mf->prev_pos_tree[i] >= 0 ) mf->prev_pos_tree[i] -= offset;
|
||||
Mf_read_block( mf );
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
bool Mf_init( struct Matchfinder * const mf,
|
||||
const int dict_size, const int len_limit, const int ifd )
|
||||
{
|
||||
const int buffer_size_limit = ( 2 * dict_size ) + before_size + after_size;
|
||||
int i;
|
||||
mf->partial_data_pos = 0;
|
||||
mf->prev_positions =
|
||||
(int32_t *)malloc( num_prev_positions * sizeof (int32_t) );
|
||||
if( !mf->prev_positions )
|
||||
{
|
||||
show_error( "Not enough memory. Try a smaller dictionary size.", 0, false );
|
||||
cleanup_and_fail( 1 );
|
||||
}
|
||||
mf->prev_positions = (int32_t *)malloc( num_prev_positions * sizeof (int32_t) );
|
||||
if( !mf->prev_positions ) return false;
|
||||
mf->pos = 0;
|
||||
mf->cyclic_pos = 0;
|
||||
mf->stream_pos = 0;
|
||||
mf->match_len_limit_ = len_limit;
|
||||
mf->match_len_limit = len_limit;
|
||||
mf->cycles = ( len_limit < max_match_len ) ? 16 + ( len_limit / 2 ) : 256;
|
||||
mf->infd = ifd;
|
||||
mf->at_stream_end = false;
|
||||
|
||||
for( i = 0; i < num_prev_positions; ++i ) mf->prev_positions[i] = -1;
|
||||
mf->buffer_size = max( 65536, dict_size );
|
||||
mf->buffer = (uint8_t *)malloc( mf->buffer_size );
|
||||
if( !mf->buffer )
|
||||
{
|
||||
show_error( "Not enough memory. Try a smaller dictionary size.", 0, false );
|
||||
cleanup_and_fail( 1 );
|
||||
}
|
||||
if( !mf->buffer ) { free( mf->prev_positions ); return false; }
|
||||
if( Mf_read_block( mf ) && !mf->at_stream_end &&
|
||||
mf->buffer_size < buffer_size_limit )
|
||||
{
|
||||
mf->buffer_size = buffer_size_limit;
|
||||
mf->buffer = (uint8_t *)realloc( mf->buffer, mf->buffer_size );
|
||||
if( !mf->buffer )
|
||||
{
|
||||
show_error( "Not enough memory. Try a smaller dictionary size.", 0, false );
|
||||
cleanup_and_fail( 1 );
|
||||
}
|
||||
uint8_t * const tmp = (uint8_t *)realloc( mf->buffer, mf->buffer_size );
|
||||
if( !tmp )
|
||||
{ free( mf->buffer ); free( mf->prev_positions ); return false; }
|
||||
mf->buffer = tmp;
|
||||
Mf_read_block( mf );
|
||||
}
|
||||
if( mf->at_stream_end && mf->stream_pos < dict_size )
|
||||
mf->dictionary_size_ = max( min_dictionary_size, mf->stream_pos );
|
||||
else mf->dictionary_size_ = dict_size;
|
||||
mf->dictionary_size = max( min_dictionary_size, mf->stream_pos );
|
||||
else mf->dictionary_size = dict_size;
|
||||
mf->pos_limit = mf->buffer_size;
|
||||
if( !mf->at_stream_end ) mf->pos_limit -= after_size;
|
||||
mf->prev_pos_tree =
|
||||
(int32_t *)malloc( 2 * mf->dictionary_size_ * sizeof (int32_t) );
|
||||
(int32_t *)malloc( 2 * mf->dictionary_size * sizeof (int32_t) );
|
||||
if( !mf->prev_pos_tree )
|
||||
{
|
||||
show_error( "Not enough memory. Try a smaller dictionary size.", 0, false );
|
||||
cleanup_and_fail( 1 );
|
||||
}
|
||||
for( i = 0; i < num_prev_positions; ++i ) mf->prev_positions[i] = -1;
|
||||
{ free( mf->buffer ); free( mf->prev_positions ); return false; }
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -116,32 +126,6 @@ void Mf_reset( struct Matchfinder * const mf )
|
|||
}
|
||||
|
||||
|
||||
void Mf_move_pos( struct Matchfinder * const mf )
|
||||
{
|
||||
if( ++mf->cyclic_pos >= mf->dictionary_size_ ) mf->cyclic_pos = 0;
|
||||
if( ++mf->pos >= mf->pos_limit )
|
||||
{
|
||||
if( mf->pos > mf->stream_pos )
|
||||
internal_error( "pos > stream_pos in Mf_move_pos" );
|
||||
if( !mf->at_stream_end )
|
||||
{
|
||||
int i;
|
||||
const int offset = mf->pos - mf->dictionary_size_ - before_size;
|
||||
const int size = mf->stream_pos - offset;
|
||||
memmove( mf->buffer, mf->buffer + offset, size );
|
||||
mf->partial_data_pos += offset;
|
||||
mf->pos -= offset;
|
||||
mf->stream_pos -= offset;
|
||||
for( i = 0; i < num_prev_positions; ++i )
|
||||
if( mf->prev_positions[i] >= 0 ) mf->prev_positions[i] -= offset;
|
||||
for( i = 0; i < 2 * mf->dictionary_size_; ++i )
|
||||
if( mf->prev_pos_tree[i] >= 0 ) mf->prev_pos_tree[i] -= offset;
|
||||
Mf_read_block( mf );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
int Mf_longest_match_len( struct Matchfinder * const mf, int * const distances )
|
||||
{
|
||||
int32_t * ptr0 = mf->prev_pos_tree + ( mf->cyclic_pos << 1 );
|
||||
|
|
@ -150,11 +134,11 @@ int Mf_longest_match_len( struct Matchfinder * const mf, int * const distances )
|
|||
const uint8_t * newdata;
|
||||
int len = 0, len0 = 0, len1 = 0;
|
||||
int maxlen = min_match_len - 1;
|
||||
const int min_pos = (mf->pos >= mf->dictionary_size_) ?
|
||||
(mf->pos - mf->dictionary_size_ + 1) : 0;
|
||||
const int min_pos = (mf->pos >= mf->dictionary_size) ?
|
||||
(mf->pos - mf->dictionary_size + 1) : 0;
|
||||
const uint8_t * const data = mf->buffer + mf->pos;
|
||||
int count, delta, key2, key3, key4, newpos, tmp;
|
||||
int len_limit = mf->match_len_limit_;
|
||||
int len_limit = mf->match_len_limit;
|
||||
|
||||
if( len_limit > Mf_available_bytes( mf ) )
|
||||
{
|
||||
|
|
@ -187,7 +171,6 @@ int Mf_longest_match_len( struct Matchfinder * const mf, int * const distances )
|
|||
newpos = mf->prev_positions[key4];
|
||||
mf->prev_positions[key4] = mf->pos;
|
||||
|
||||
|
||||
for( count = mf->cycles; ; )
|
||||
{
|
||||
if( newpos < min_pos || --count < 0 ) { *ptr0 = *ptr1 = -1; break; }
|
||||
|
|
@ -199,7 +182,7 @@ int Mf_longest_match_len( struct Matchfinder * const mf, int * const distances )
|
|||
|
||||
newptr = mf->prev_pos_tree +
|
||||
( ( mf->cyclic_pos - delta +
|
||||
( ( mf->cyclic_pos >= delta ) ? 0 : mf->dictionary_size_ ) ) << 1 );
|
||||
( ( mf->cyclic_pos >= delta ) ? 0 : mf->dictionary_size ) ) << 1 );
|
||||
|
||||
if( len < len_limit )
|
||||
{
|
||||
|
|
@ -234,42 +217,44 @@ int Mf_longest_match_len( struct Matchfinder * const mf, int * const distances )
|
|||
}
|
||||
|
||||
|
||||
void Re_flush_data( struct Range_encoder * const range_encoder )
|
||||
void Re_flush_data( struct Range_encoder * const renc )
|
||||
{
|
||||
if( range_encoder->pos > 0 )
|
||||
if( renc->pos > 0 )
|
||||
{
|
||||
if( range_encoder->outfd >= 0 &&
|
||||
writeblock( range_encoder->outfd, range_encoder->buffer,
|
||||
range_encoder->pos ) != range_encoder->pos )
|
||||
if( renc->outfd >= 0 &&
|
||||
writeblock( renc->outfd, renc->buffer,
|
||||
renc->pos ) != renc->pos )
|
||||
{ show_error( "Write error", errno, false ); cleanup_and_fail( 1 ); }
|
||||
range_encoder->partial_member_pos += range_encoder->pos;
|
||||
range_encoder->pos = 0;
|
||||
renc->partial_member_pos += renc->pos;
|
||||
renc->pos = 0;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void Lee_encode( struct Len_encoder * const len_encoder,
|
||||
struct Range_encoder * const range_encoder,
|
||||
struct Range_encoder * const renc,
|
||||
int symbol, const int pos_state )
|
||||
{
|
||||
symbol -= min_match_len;
|
||||
if( symbol < len_low_symbols )
|
||||
{
|
||||
Re_encode_bit( range_encoder, &len_encoder->choice1, 0 );
|
||||
Re_encode_tree( range_encoder, len_encoder->bm_low[pos_state], symbol, len_low_bits );
|
||||
Re_encode_bit( renc, &len_encoder->choice1, 0 );
|
||||
Re_encode_tree( renc, len_encoder->bm_low[pos_state], symbol, len_low_bits );
|
||||
}
|
||||
else
|
||||
{
|
||||
Re_encode_bit( range_encoder, &len_encoder->choice1, 1 );
|
||||
Re_encode_bit( renc, &len_encoder->choice1, 1 );
|
||||
if( symbol < len_low_symbols + len_mid_symbols )
|
||||
{
|
||||
Re_encode_bit( range_encoder, &len_encoder->choice2, 0 );
|
||||
Re_encode_tree( range_encoder, len_encoder->bm_mid[pos_state], symbol - len_low_symbols, len_mid_bits );
|
||||
Re_encode_bit( renc, &len_encoder->choice2, 0 );
|
||||
Re_encode_tree( renc, len_encoder->bm_mid[pos_state],
|
||||
symbol - len_low_symbols, len_mid_bits );
|
||||
}
|
||||
else
|
||||
{
|
||||
Re_encode_bit( range_encoder, &len_encoder->choice2, 1 );
|
||||
Re_encode_tree( range_encoder, len_encoder->bm_high, symbol - len_low_symbols - len_mid_symbols, len_high_bits );
|
||||
Re_encode_bit( renc, &len_encoder->choice2, 1 );
|
||||
Re_encode_tree( renc, len_encoder->bm_high,
|
||||
symbol - len_low_symbols - len_mid_symbols, len_high_bits );
|
||||
}
|
||||
}
|
||||
if( --len_encoder->counters[pos_state] <= 0 )
|
||||
|
|
@ -277,11 +262,32 @@ void Lee_encode( struct Len_encoder * const len_encoder,
|
|||
}
|
||||
|
||||
|
||||
/* End Of Stream mark => (dis == 0xFFFFFFFFU, len == min_match_len) */
|
||||
void LZe_full_flush( struct LZ_encoder * const encoder, const State state )
|
||||
{
|
||||
int i;
|
||||
const int pos_state = Mf_data_position( encoder->matchfinder ) & pos_state_mask;
|
||||
File_trailer trailer;
|
||||
Re_encode_bit( &encoder->range_encoder, &encoder->bm_match[state][pos_state], 1 );
|
||||
Re_encode_bit( &encoder->range_encoder, &encoder->bm_rep[state], 0 );
|
||||
LZe_encode_pair( encoder, 0xFFFFFFFFU, min_match_len, pos_state );
|
||||
Re_flush( &encoder->range_encoder );
|
||||
Ft_set_data_crc( trailer, LZe_crc( encoder ) );
|
||||
Ft_set_data_size( trailer, Mf_data_position( encoder->matchfinder ) );
|
||||
Ft_set_member_size( trailer, Re_member_position( &encoder->range_encoder ) +
|
||||
Ft_size );
|
||||
for( i = 0; i < Ft_size; ++i )
|
||||
Re_put_byte( &encoder->range_encoder, trailer[i] );
|
||||
Re_flush_data( &encoder->range_encoder );
|
||||
}
|
||||
|
||||
|
||||
void LZe_fill_align_prices( struct LZ_encoder * const encoder )
|
||||
{
|
||||
int i;
|
||||
for( i = 0; i < dis_align_size; ++i )
|
||||
encoder->align_prices[i] = price_symbol_reversed( encoder->bm_align, i, dis_align_bits );
|
||||
encoder->align_prices[i] =
|
||||
price_symbol_reversed( encoder->bm_align, i, dis_align_bits );
|
||||
encoder->align_price_count = dis_align_size;
|
||||
}
|
||||
|
||||
|
|
@ -295,7 +301,8 @@ void LZe_fill_distance_prices( struct LZ_encoder * const encoder )
|
|||
const int direct_bits = ( dis_slot >> 1 ) - 1;
|
||||
const int base = ( 2 | ( dis_slot & 1 ) ) << direct_bits;
|
||||
const int price =
|
||||
price_symbol_reversed( encoder->bm_dis + base - dis_slot, dis - base, direct_bits );
|
||||
price_symbol_reversed( encoder->bm_dis + base - dis_slot,
|
||||
dis - base, direct_bits );
|
||||
for( dis_state = 0; dis_state < max_dis_states; ++dis_state )
|
||||
encoder->dis_prices[dis_state][dis] = price;
|
||||
}
|
||||
|
|
@ -320,17 +327,63 @@ void LZe_fill_distance_prices( struct LZ_encoder * const encoder )
|
|||
}
|
||||
|
||||
|
||||
bool LZe_init( struct LZ_encoder * const encoder,
|
||||
struct Matchfinder * const mf,
|
||||
const File_header header, const int outfd )
|
||||
{
|
||||
int i, j;
|
||||
encoder->longest_match_found = 0;
|
||||
encoder->crc = 0xFFFFFFFFU;
|
||||
|
||||
for( i = 0; i < states; ++i )
|
||||
{
|
||||
for( j = 0; j < pos_states; ++j )
|
||||
{
|
||||
Bm_init( &encoder->bm_match[i][j] );
|
||||
Bm_init( &encoder->bm_len[i][j] );
|
||||
}
|
||||
Bm_init( &encoder->bm_rep[i] );
|
||||
Bm_init( &encoder->bm_rep0[i] );
|
||||
Bm_init( &encoder->bm_rep1[i] );
|
||||
Bm_init( &encoder->bm_rep2[i] );
|
||||
}
|
||||
for( i = 0; i < max_dis_states; ++i )
|
||||
for( j = 0; j < 1<<dis_slot_bits; ++j )
|
||||
Bm_init( &encoder->bm_dis_slot[i][j] );
|
||||
for( i = 0; i < modeled_distances-end_dis_model+1; ++i )
|
||||
Bm_init( &encoder->bm_dis[i] );
|
||||
for( i = 0; i < dis_align_size; ++i )
|
||||
Bm_init( &encoder->bm_align[i] );
|
||||
|
||||
encoder->matchfinder = mf;
|
||||
if( !Re_init( &encoder->range_encoder, outfd ) ) return false;
|
||||
Lee_init( &encoder->len_encoder, encoder->matchfinder->match_len_limit );
|
||||
Lee_init( &encoder->rep_match_len_encoder, encoder->matchfinder->match_len_limit );
|
||||
Lie_init( &encoder->literal_encoder );
|
||||
encoder->num_dis_slots =
|
||||
2 * real_bits( encoder->matchfinder->dictionary_size - 1 );
|
||||
|
||||
LZe_fill_align_prices( encoder );
|
||||
|
||||
for( i = 0; i < Fh_size; ++i )
|
||||
Re_put_byte( &encoder->range_encoder, header[i] );
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
/* Return value == number of bytes advanced (ahead).
|
||||
trials[0]..trials[retval-1] contain the steps to encode.
|
||||
( trials[0].dis == -1 && trials[0].price == 1 ) means literal. */
|
||||
( trials[0].dis == -1 && trials[0].price == 1 ) means literal.
|
||||
*/
|
||||
int LZe_sequence_optimizer( struct LZ_encoder * const encoder,
|
||||
const int reps[num_rep_distances], const State state )
|
||||
const int reps[num_rep_distances],
|
||||
const State state )
|
||||
{
|
||||
int main_len, i, rep, cur = 0, num_trials;
|
||||
int replens[num_rep_distances];
|
||||
int rep_index = 0;
|
||||
|
||||
if( encoder->longest_match_found > 0 ) /* from previous call */
|
||||
if( encoder->longest_match_found > 0 ) /* from previous call */
|
||||
{
|
||||
main_len = encoder->longest_match_found;
|
||||
encoder->longest_match_found = 0;
|
||||
|
|
@ -342,20 +395,21 @@ int LZe_sequence_optimizer( struct LZ_encoder * const encoder,
|
|||
replens[i] = Mf_true_match_len( encoder->matchfinder, 0, reps[i] + 1, max_match_len );
|
||||
if( replens[i] > replens[rep_index] ) rep_index = i;
|
||||
}
|
||||
if( replens[rep_index] >= Mf_match_len_limit( encoder->matchfinder ) )
|
||||
if( replens[rep_index] >= encoder->matchfinder->match_len_limit )
|
||||
{
|
||||
encoder->trials[0].dis = rep_index;
|
||||
encoder->trials[0].price = replens[rep_index];
|
||||
LZe_move_pos( encoder, replens[rep_index], true );
|
||||
LZe_move_pos( encoder, replens[rep_index] );
|
||||
return replens[rep_index];
|
||||
}
|
||||
|
||||
if( main_len >= Mf_match_len_limit( encoder->matchfinder ) )
|
||||
if( main_len >= encoder->matchfinder->match_len_limit )
|
||||
{
|
||||
encoder->trials[0].dis = encoder->match_distances[Mf_match_len_limit( encoder->matchfinder )] +
|
||||
num_rep_distances;
|
||||
encoder->trials[0].dis =
|
||||
encoder->match_distances[encoder->matchfinder->match_len_limit] +
|
||||
num_rep_distances;
|
||||
encoder->trials[0].price = main_len;
|
||||
LZe_move_pos( encoder, main_len, true );
|
||||
LZe_move_pos( encoder, main_len );
|
||||
return main_len;
|
||||
}
|
||||
|
||||
|
|
@ -368,17 +422,21 @@ int LZe_sequence_optimizer( struct LZ_encoder * const encoder,
|
|||
const uint8_t match_byte = Mf_peek( encoder->matchfinder, -reps[0]-1 );
|
||||
|
||||
encoder->trials[0].state = state;
|
||||
for( i = 0; i < num_rep_distances; ++i ) encoder->trials[0].reps[i] = reps[i];
|
||||
for( i = 0; i < num_rep_distances; ++i )
|
||||
encoder->trials[0].reps[i] = reps[i];
|
||||
encoder->trials[1].dis = -1;
|
||||
encoder->trials[1].prev_index = 0;
|
||||
encoder->trials[1].price = price0( encoder->bm_match[state][pos_state] );
|
||||
if( St_is_char( state ) )
|
||||
encoder->trials[1].price += Lie_price_symbol( &encoder->literal_encoder, prev_byte, cur_byte );
|
||||
encoder->trials[1].price +=
|
||||
Lie_price_symbol( &encoder->literal_encoder, prev_byte, cur_byte );
|
||||
else
|
||||
encoder->trials[1].price += Lie_price_matched( &encoder->literal_encoder, prev_byte, cur_byte, match_byte );
|
||||
encoder->trials[1].price +=
|
||||
Lie_price_matched( &encoder->literal_encoder, prev_byte, cur_byte, match_byte );
|
||||
|
||||
if( match_byte == cur_byte )
|
||||
Tr_update( &encoder->trials[1], 0, 0, rep_match_price + LZe_price_rep_len1( encoder, state, pos_state ) );
|
||||
Tr_update( &encoder->trials[1], 0, 0, rep_match_price +
|
||||
LZe_price_rep_len1( encoder, state, pos_state ) );
|
||||
|
||||
if( main_len < min_match_len )
|
||||
{
|
||||
|
|
@ -404,7 +462,7 @@ int LZe_sequence_optimizer( struct LZ_encoder * const encoder,
|
|||
encoder->trials[len].dis = encoder->match_distances[len] + num_rep_distances;
|
||||
encoder->trials[len].prev_index = 0;
|
||||
encoder->trials[len].price = normal_match_price +
|
||||
LZe_price_pair( encoder, encoder->match_distances[len], len, pos_state );
|
||||
LZe_price_pair( encoder, encoder->match_distances[len], len, pos_state );
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -435,7 +493,7 @@ int LZe_sequence_optimizer( struct LZ_encoder * const encoder,
|
|||
return cur;
|
||||
}
|
||||
newlen = LZe_read_match_distances( encoder );
|
||||
if( newlen >= Mf_match_len_limit( encoder->matchfinder ) )
|
||||
if( newlen >= encoder->matchfinder->match_len_limit )
|
||||
{
|
||||
encoder->longest_match_found = newlen;
|
||||
LZe_backward( encoder, cur );
|
||||
|
|
@ -449,6 +507,7 @@ int LZe_sequence_optimizer( struct LZ_encoder * const encoder,
|
|||
|
||||
for( i = 0; i < num_rep_distances; ++i )
|
||||
cur_trial->reps[i] = encoder->trials[prev_index].reps[i];
|
||||
|
||||
if( prev_index == cur - 1 )
|
||||
{
|
||||
if( cur_trial->dis == 0 ) St_set_short_rep( &cur_trial->state );
|
||||
|
|
@ -466,11 +525,14 @@ int LZe_sequence_optimizer( struct LZ_encoder * const encoder,
|
|||
cur_byte = Mf_peek( encoder->matchfinder, 0 );
|
||||
match_byte = Mf_peek( encoder->matchfinder, -cur_trial->reps[0]-1 );
|
||||
|
||||
next_price = cur_trial->price + price0( encoder->bm_match[cur_trial->state][pos_state] );
|
||||
next_price = cur_trial->price +
|
||||
price0( encoder->bm_match[cur_trial->state][pos_state] );
|
||||
if( St_is_char( cur_trial->state ) )
|
||||
next_price += Lie_price_symbol( &encoder->literal_encoder, prev_byte, cur_byte );
|
||||
next_price += Lie_price_symbol( &encoder->literal_encoder,
|
||||
prev_byte, cur_byte );
|
||||
else
|
||||
next_price += Lie_price_matched( &encoder->literal_encoder, prev_byte, cur_byte, match_byte );
|
||||
next_price += Lie_price_matched( &encoder->literal_encoder,
|
||||
prev_byte, cur_byte, match_byte );
|
||||
Mf_move_pos( encoder->matchfinder );
|
||||
|
||||
next_trial = &encoder->trials[cur+1];
|
||||
|
|
@ -486,7 +548,7 @@ int LZe_sequence_optimizer( struct LZ_encoder * const encoder,
|
|||
|
||||
len_limit = min( min( max_num_trials - 1 - cur,
|
||||
Mf_available_bytes( encoder->matchfinder ) ),
|
||||
Mf_match_len_limit( encoder->matchfinder ) );
|
||||
encoder->matchfinder->match_len_limit );
|
||||
if( len_limit < min_match_len ) continue;
|
||||
|
||||
for( rep = 0; rep < num_rep_distances; ++rep )
|
||||
|
|
@ -545,82 +607,21 @@ int LZe_sequence_optimizer( struct LZ_encoder * const encoder,
|
|||
}
|
||||
|
||||
|
||||
/* End Of Stream mark => (dis == 0xFFFFFFFFU, len == min_match_len) */
|
||||
void LZe_full_flush( struct LZ_encoder * const encoder, const State state )
|
||||
{
|
||||
int i;
|
||||
const int pos_state = Mf_data_position( encoder->matchfinder ) & pos_state_mask;
|
||||
File_trailer trailer;
|
||||
Re_encode_bit( &encoder->range_encoder, &encoder->bm_match[state][pos_state], 1 );
|
||||
Re_encode_bit( &encoder->range_encoder, &encoder->bm_rep[state], 0 );
|
||||
LZe_encode_pair( encoder, 0xFFFFFFFFU, min_match_len, pos_state );
|
||||
Re_flush( &encoder->range_encoder );
|
||||
Ft_set_data_crc( trailer, LZe_crc( encoder ) );
|
||||
Ft_set_data_size( trailer, Mf_data_position( encoder->matchfinder ) );
|
||||
Ft_set_member_size( trailer, LZe_member_position( encoder ) + Ft_size );
|
||||
for( i = 0; i < Ft_size; ++i )
|
||||
Re_put_byte( &encoder->range_encoder, trailer[i] );
|
||||
Re_flush_data( &encoder->range_encoder );
|
||||
}
|
||||
|
||||
|
||||
void LZe_init( struct LZ_encoder * const encoder, struct Matchfinder * const mf,
|
||||
const File_header header, const int outfd )
|
||||
{
|
||||
int i, j;
|
||||
encoder->longest_match_found = 0;
|
||||
encoder->crc_ = 0xFFFFFFFFU;
|
||||
|
||||
for( i = 0; i < states; ++i )
|
||||
{
|
||||
for( j = 0; j < pos_states; ++j )
|
||||
{
|
||||
Bm_init( &encoder->bm_match[i][j] );
|
||||
Bm_init( &encoder->bm_len[i][j] );
|
||||
}
|
||||
Bm_init( &encoder->bm_rep[i] );
|
||||
Bm_init( &encoder->bm_rep0[i] );
|
||||
Bm_init( &encoder->bm_rep1[i] );
|
||||
Bm_init( &encoder->bm_rep2[i] );
|
||||
}
|
||||
for( i = 0; i < max_dis_states; ++i )
|
||||
for( j = 0; j < 1<<dis_slot_bits; ++j )
|
||||
Bm_init( &encoder->bm_dis_slot[i][j] );
|
||||
for( i = 0; i < modeled_distances-end_dis_model+1; ++i )
|
||||
Bm_init( &encoder->bm_dis[i] );
|
||||
for( i = 0; i < dis_align_size; ++i )
|
||||
Bm_init( &encoder->bm_align[i] );
|
||||
|
||||
encoder->matchfinder = mf;
|
||||
Re_init( &encoder->range_encoder, outfd );
|
||||
Lee_init( &encoder->len_encoder, Mf_match_len_limit( encoder->matchfinder ) ),
|
||||
Lee_init( &encoder->rep_match_len_encoder, Mf_match_len_limit( encoder->matchfinder ) ),
|
||||
Lie_init( &encoder->literal_encoder );
|
||||
encoder->num_dis_slots = 2 * real_bits( Mf_dictionary_size( encoder->matchfinder ) - 1 );
|
||||
|
||||
LZe_fill_align_prices( encoder );
|
||||
|
||||
for( i = 0; i < Fh_size; ++i )
|
||||
Re_put_byte( &encoder->range_encoder, header[i] );
|
||||
}
|
||||
|
||||
|
||||
bool LZe_encode_member( struct LZ_encoder * const encoder,
|
||||
const long long member_size )
|
||||
{
|
||||
const long long member_size_limit =
|
||||
member_size - Ft_size - max_marker_size;
|
||||
const int fill_count =
|
||||
( Mf_match_len_limit( encoder->matchfinder ) > 12 ) ? 512 : 2048;
|
||||
( encoder->matchfinder->match_len_limit > 12 ) ? 512 : 2048;
|
||||
int fill_counter = 0;
|
||||
int ahead;
|
||||
int i;
|
||||
int ahead, i;
|
||||
int rep_distances[num_rep_distances];
|
||||
State state = 0;
|
||||
for( i = 0; i < num_rep_distances; ++i ) rep_distances[i] = 0;
|
||||
|
||||
if( Mf_data_position( encoder->matchfinder ) != 0 ||
|
||||
LZe_member_position( encoder ) != Fh_size )
|
||||
Re_member_position( &encoder->range_encoder ) != Fh_size )
|
||||
return false; /* can be called only once */
|
||||
|
||||
if( !Mf_finished( encoder->matchfinder ) ) /* encode first byte */
|
||||
|
|
@ -629,46 +630,50 @@ bool LZe_encode_member( struct LZ_encoder * const encoder,
|
|||
const uint8_t cur_byte = Mf_peek( encoder->matchfinder, 0 );
|
||||
Re_encode_bit( &encoder->range_encoder, &encoder->bm_match[state][0], 0 );
|
||||
Lie_encode( &encoder->literal_encoder, &encoder->range_encoder, prev_byte, cur_byte );
|
||||
CRC32_update_byte( &encoder->crc_, cur_byte );
|
||||
LZe_move_pos( encoder, 1, false );
|
||||
CRC32_update_byte( &encoder->crc, cur_byte );
|
||||
Mf_longest_match_len( encoder->matchfinder, 0 );
|
||||
Mf_move_pos( encoder->matchfinder );
|
||||
}
|
||||
|
||||
while( true )
|
||||
while( !Mf_finished( encoder->matchfinder ) )
|
||||
{
|
||||
if( Mf_finished( encoder->matchfinder ) )
|
||||
{ LZe_full_flush( encoder, state ); return true; }
|
||||
if( fill_counter <= 0 )
|
||||
{ LZe_fill_distance_prices( encoder ); fill_counter = fill_count; }
|
||||
|
||||
ahead = LZe_sequence_optimizer( encoder, rep_distances, state );
|
||||
if( ahead <= 0 ) return false;
|
||||
if( ahead <= 0 ) return false; /* can't happen */
|
||||
fill_counter -= ahead;
|
||||
|
||||
for( i = 0; ; )
|
||||
{
|
||||
const int pos_state = ( Mf_data_position( encoder->matchfinder ) - ahead ) & pos_state_mask;
|
||||
const int pos_state =
|
||||
( Mf_data_position( encoder->matchfinder ) - ahead ) & pos_state_mask;
|
||||
const int dis = encoder->trials[i].dis;
|
||||
const int len = encoder->trials[i].price;
|
||||
|
||||
bool bit = ( dis < 0 && len == 1 );
|
||||
Re_encode_bit( &encoder->range_encoder, &encoder->bm_match[state][pos_state], !bit );
|
||||
if( bit ) /* literal byte */
|
||||
Re_encode_bit( &encoder->range_encoder,
|
||||
&encoder->bm_match[state][pos_state], !bit );
|
||||
if( bit ) /* literal byte */
|
||||
{
|
||||
const uint8_t prev_byte = Mf_peek( encoder->matchfinder, -ahead-1 );
|
||||
const uint8_t cur_byte = Mf_peek( encoder->matchfinder, -ahead );
|
||||
CRC32_update_byte( &encoder->crc_, cur_byte );
|
||||
CRC32_update_byte( &encoder->crc, cur_byte );
|
||||
if( St_is_char( state ) )
|
||||
Lie_encode( &encoder->literal_encoder, &encoder->range_encoder, prev_byte, cur_byte );
|
||||
Lie_encode( &encoder->literal_encoder, &encoder->range_encoder,
|
||||
prev_byte, cur_byte );
|
||||
else
|
||||
{
|
||||
const uint8_t match_byte = Mf_peek( encoder->matchfinder, -ahead-rep_distances[0]-1 );
|
||||
Lie_encode_matched( &encoder->literal_encoder, &encoder->range_encoder, prev_byte, cur_byte, match_byte );
|
||||
const uint8_t match_byte =
|
||||
Mf_peek( encoder->matchfinder, -ahead-rep_distances[0]-1 );
|
||||
Lie_encode_matched( &encoder->literal_encoder, &encoder->range_encoder,
|
||||
prev_byte, cur_byte, match_byte );
|
||||
}
|
||||
St_set_char( &state );
|
||||
}
|
||||
else /* match or repeated match */
|
||||
{
|
||||
CRC32_update_buf( &encoder->crc_, Mf_ptr_to_current_pos( encoder->matchfinder ) - ahead, len );
|
||||
CRC32_update_buf( &encoder->crc, Mf_ptr_to_current_pos( encoder->matchfinder ) - ahead, len );
|
||||
LZe_mtf_reps( dis, rep_distances );
|
||||
bit = ( dis < num_rep_distances );
|
||||
Re_encode_bit( &encoder->range_encoder, &encoder->bm_rep[state], bit );
|
||||
|
|
@ -698,7 +703,7 @@ bool LZe_encode_member( struct LZ_encoder * const encoder,
|
|||
}
|
||||
}
|
||||
ahead -= len; i += len;
|
||||
if( LZe_member_position( encoder ) >= member_size_limit )
|
||||
if( Re_member_position( &encoder->range_encoder ) >= member_size_limit )
|
||||
{
|
||||
if( !Mf_dec_pos( encoder->matchfinder, ahead ) ) return false;
|
||||
LZe_full_flush( encoder, state );
|
||||
|
|
@ -707,4 +712,6 @@ bool LZe_encode_member( struct LZ_encoder * const encoder,
|
|||
if( ahead <= 0 ) break;
|
||||
}
|
||||
}
|
||||
LZe_full_flush( encoder, state );
|
||||
return true;
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue