67 template <
typename TIntegral>
71 using TUI =
typename std::make_unsigned<TIntegral>::type;
86 #if !DOXYGEN && ALIB_DEBUG_ARRAY_COMPRESSION
87 bool dbgIsCheckRead =
false;
96 firstVal=
const_cast<char*
>(
reinterpret_cast<const char*
>(arrayStart) );
100 min= (std::numeric_limits<TUI>::max)();
101 max= (std::numeric_limits<TUI>::min)();
112 Array(
const TIntegral& firstValue,
const TIntegral& secondValue,
size_t length )
114 firstVal=
const_cast<char*
>(
reinterpret_cast<const char*
>(&firstValue) );
115 distance= size_t(
reinterpret_cast<const char*
>(&secondValue)
116 -
reinterpret_cast<const char*
>(&firstValue)) ;
119 min= (std::numeric_limits<TUI>::max)();
120 max= (std::numeric_limits<TUI>::min)();
134 if constexpr ( std::is_unsigned<TIntegral>::value )
138 return val >= 0 ?
TUI( val << 1 )
139 :
TUI( ((-val - 1) << 1) | 1 );
147 #if ALIB_DEBUG_ARRAY_COMPRESSION
155 if constexpr ( std::is_unsigned<TIntegral>::value )
156 *(
reinterpret_cast<TIntegral*
>(
firstVal + idx *
distance))= TIntegral(value);
159 (value & 1) == 0 ? TIntegral( value >> 1)
160 : TIntegral( -( TIntegral( value >> 1) + 1 ) );
163 #if ALIB_DEBUG_ARRAY_COMPRESSION
166 "BITBUFFER/AC",
"Error reading back compressed array data" )
188 minInc= (std::numeric_limits<TUI>::max)();
189 minDec= (std::numeric_limits<TUI>::max)();
192 min= (std::min)(
min, prevVal );
193 max= (std::max)(
max, prevVal );
195 for(
size_t i= 1; i <
len; ++i) {
197 min= (std::min)(
min, val );
198 max= (std::max)(
max, val );
372 template <
typename TValue>
397 template <
typename TValue>
421#include "alib/bitbuffer/ac_v1/acalgos.hpp.inl"
428template <
typename TValue>
432 Algorithm algorithmsToTry,
433 Statistics* statistics ) {
435 "BitBuffer is smaller than uncompressed data."
436 " No buffer overflow checks are performed during compression." )
438 "BitBuffer remaining size should be twice as large as uncompressed data."
439 " No buffer overflow checks are performed during compression." )
441auto initialBufferState= bw.GetIndex();
442auto initialBufferFill = bw.Usage();
443int multipleAlgorithms= CountElements(algorithmsToTry) > 1;
444ALIB_ASSERT_ERROR(
int(algorithmsToTry) != 0,
"BITBUFFER/AC",
"No algorithms to check given" )
447auto bestAlgoNo= (std::numeric_limits<
int>::max)();
449auto leastBits = (std::numeric_limits<
size_t>::max)();
451bool isFirstAlgo = true;
457 if ( !HasBits(algorithmsToTry, algo ) )
460 bw.Reset(initialBufferState);
465 if( multipleAlgorithms )
466 bw.WriteBits<3>( algoNo );
469 switch( lastAlgo= algo ) {
470 case Algorithm::Uncompressed: writeUncompressed(bw, data); break;
471 case Algorithm::MinMax: writeMinMax (bw, data); break;
472 case Algorithm::Sparse: writeSparse (bw, data); break;
473 case Algorithm::VerySparse: writeVerySparse (bw, data); break;
474 case Algorithm::Incremental: writeIncremental (bw, data); break;
475 case Algorithm::Huffman: writeHuffman (bw, data); break;
476 default: ALIB_ERROR(
"BITBUFFER/AC",
477 "Internal error: Unknown compression algorithm number read") break;
480 auto bufferFill= bw.Usage();
483 statistics->writeTimes [algoNo]+= tm.Age();
484 statistics->sumCompressed[algoNo]+= (bufferFill - initialBufferFill)/8;
488 "Array compression: Nothing written")
489 if( leastBits > bufferFill - initialBufferFill ) {
490 leastBits= bufferFill - initialBufferFill;
496 #if ALIB_DEBUG_ARRAY_COMPRESSION
499 BitReader br(bw.GetBuffer(), initialBufferState);
500 if( multipleAlgorithms ) {
501 auto readBackAlgo=
Algorithm( 1 << br.ReadBits<3>() );
503 "Wrong algorithm id was read back. This must never happen." )
506 data.dbgIsCheckRead=
true;
517 "Internal error: Unknown compression algorithm number read")
522 data.dbgIsCheckRead= false;
526 if( !multipleAlgorithms )
531 statistics->ctdCompressions++;
532 statistics->sumUncompressed+= data.length() *
sizeof(TValue);
533 statistics->ctdWins [bestAlgoNo]++;
534 statistics->sumCompressedWon [bestAlgoNo]+= leastBits/8;
535 statistics->sumUnCompressedWon[bestAlgoNo]+= data.length() *
sizeof(TValue);
540if( multipleAlgorithms && (bestAlgo != lastAlgo) ) {
541 bw.Reset( initialBufferState );
542 bw.WriteBits<3>( bestAlgoNo );
551 default:
ALIB_ERROR(
"BITBUFFER/AC",
"Internal error: Unknown compression "
552 "algorithm number read") break;
558return std::make_pair( leastBits, bestAlgo );
561template <typename TValue>
567 "No algorithms to check given" )
568 bool multipleAlgorithms= CountElements(algorithmsToTry) > 1;
571 auto algo= multipleAlgorithms ?
Algorithm( 1 << br.ReadBits<3>() )
581 default:
ALIB_ERROR(
"BITBUFFER",
"Internal error: Unknown compression "
582 "algorithm number read") break;
588 statistics->readTimes[algoNo]+= tm.Age();
589 statistics->ctdReads [algoNo]++;
#define ALIB_ALLOW_SPARSE_ENUM_SWITCH
#define ALIB_ASSERT_WARNING(cond, domain,...)
#define ALIB_ERROR(domain,...)
#define ALIB_POP_ALLOWANCE
#define ALIB_ASSERT_ERROR(cond, domain,...)
#define ALIB_BOXING_VTABLE_DECLARE(TMapped, Identifier)
Reads bits from a #"BitBufferBase".
Writes bits into a #"BitBufferBase".
TUI maxDec
Maximum decrease between two adjacent values.
size_t distance
The distance in memory between two array values.
char * firstVal
Pointer to the first array value.
TUI max
Maximum value (when zig-zag encoded).
void set(size_t idx, TUI value)
TUI maxInc
Maximum increase between two adjacent values.
TUI min
Maximum value (when zig-zag encoded).
TUI minInc
Minimum increase between two adjacent values.
Array(const TIntegral &firstValue, const TIntegral &secondValue, size_t length)
TUI get(size_t idx) const
size_t len
The length of the array.
TUI minDec
Minimum decrease between two adjacent values.
Array(const TIntegral *arrayStart, size_t length)
typename std::make_unsigned< TIntegral >::type TUI
The unsigned version of template type TIntegral.
ArrayCompressor()=delete
Deleted default constructor (this class cannot be created).
static std::pair< size_t, Algorithm > Compress(BitWriter &bitWriter, Array< TValue > &data, Algorithm algorithmsToTry=Algorithm::ALL, Statistics *statistics=nullptr)
static void Decompress(BitReader &bitReader, Array< TValue > &data, Algorithm algorithm=Algorithm::ALL, Statistics *statistics=nullptr)
static constexpr int NumberOfAlgorithms
The number of algorithms implemented.
@ Huffman
Huffman encoding (byte based).
@ Incremental
Only distances of the values are written.
@ ALL
All compression methods selected.
@ Sparse
Writes '1' if next value is equal to previous, '0' plus next value otherwise.
@ MinMax
Stores the differences between the minimum and maximum value found.
@ VerySparse
Writes the number of following equal or non equal values.
@ NONE
No compression method selected.
#define ALIB_ENUMS_MAKE_BITWISE(TEnum)
#define ALIB_ENUMS_MAKE_ITERABLE(TEnum, StopElement)
#define ALIB_ENUMS_ASSIGN_RECORD(TEnum, TRecord)
constexpr std::underlying_type< TEnum >::type ToSequentialEnumeration(TEnum element)
bitbuffer::BitReader BitReader
Type alias in namespace #"%alib".
strings::TString< character > String
Type alias in namespace #"%alib".
time::Ticks Ticks
Type alias in namespace #"%alib".
bitbuffer::BitWriter BitWriter
Type alias in namespace #"%alib".
strings::TAString< character, lang::HeapAllocator > AString
Type alias in namespace #"%alib".
enumops::EnumIterator< TEnum > EnumIterator
Type alias in namespace #"%alib".
int ctdCompressions
The number of executed compressions.
size_t sumUnCompressedWon[NumberOfAlgorithms]
size_t sumCompressed[NumberOfAlgorithms]
For each algorithm, the sum of resulting bytes of all compressions performed.
int ctdWins[NumberOfAlgorithms]
Ticks::Duration readTimes[NumberOfAlgorithms]
The overall decompression time of each algorithm.
size_t sumCompressedWon[NumberOfAlgorithms]
int ctdReads[NumberOfAlgorithms]
The number of measured decompression runs of each algorithm.
Statistics & operator+=(const Statistics &other)
void Print(AString &result, const String &headline, bool printTotals)
size_t sumUncompressed
The overall given array data to compress.
Ticks::Duration writeTimes[NumberOfAlgorithms]
The overall compression time of each algorithm.