68 template <
typename TIntegral>
73 using TUI =
typename std::make_unsigned<TIntegral>::type;
88 #if !DOXYGEN && ALIB_DEBUG_ARRAY_COMPRESSION
89 bool dbgIsCheckRead =
false;
99 reinterpret_cast<const char*
>(arrayStart) );
103 min= (std::numeric_limits<TUI>::max)();
104 max= (std::numeric_limits<TUI>::min)();
115 Array(
const TIntegral& firstValue,
const TIntegral& secondValue,
size_t length )
118 reinterpret_cast<const char*
>(&firstValue) );
119 distance= size_t(
reinterpret_cast<const char*
>(&secondValue)
120 -
reinterpret_cast<const char*
>(&firstValue)) ;
123 min= (std::numeric_limits<TUI>::max)();
124 max= (std::numeric_limits<TUI>::min)();
138 if constexpr ( std::is_unsigned<TIntegral>::value )
142 return val >= 0 ?
TUI( val << 1 )
143 :
TUI( ((-val - 1) << 1) | 1 );
151 #if ALIB_DEBUG_ARRAY_COMPRESSION
159 if constexpr ( std::is_unsigned<TIntegral>::value )
160 *(
reinterpret_cast<TIntegral*
>(
firstVal + idx *
distance))= TIntegral(value);
163 (value & 1) == 0 ? TIntegral( value >> 1)
164 : TIntegral( -( TIntegral( value >> 1) + 1 ) );
167 #if ALIB_DEBUG_ARRAY_COMPRESSION
170 "BITBUFFER/AC",
"Error reading back compressed array data" )
192 minInc= (std::numeric_limits<TUI>::max)();
193 minDec= (std::numeric_limits<TUI>::max)();
196 min= (std::min)(
min, prevVal );
197 max= (std::max)(
max, prevVal );
199 for(
size_t i= 1; i <
len; ++i) {
201 min= (std::min)(
min, val );
202 max= (std::max)(
max, val );
377 template <
typename TValue>
402 template <
typename TValue>
423#include "alib/bitbuffer/ac_v1/acalgos.inl.inl"
430template <
typename TValue>
434 Algorithm algorithmsToTry,
435 Statistics* statistics ) {
437 "BitBuffer is smaller than uncompressed data."
438 " No buffer overflow checks are performed during compression." )
440 "BitBuffer remaining size should be twice as large as uncompressed data."
441 " No buffer overflow checks are performed during compression." )
443auto initialBufferState= bw.GetIndex();
444auto initialBufferFill = bw.Usage();
445int multipleAlgorithms= CountElements(algorithmsToTry) > 1;
446ALIB_ASSERT_ERROR(
int(algorithmsToTry) != 0,
"BITBUFFER/AC",
"No algorithms to check given" )
449auto bestAlgoNo= (std::numeric_limits<
int>::max)();
451auto leastBits = (std::numeric_limits<
size_t>::max)();
453bool isFirstAlgo = true;
459 if ( !HasBits(algorithmsToTry, algo ) )
462 bw.Reset(initialBufferState);
467 if( multipleAlgorithms )
468 bw.Write<3>( algoNo );
471 switch( lastAlgo= algo ) {
472 case Algorithm::Uncompressed: writeUncompressed(bw, data); break;
473 case Algorithm::MinMax: writeMinMax (bw, data); break;
474 case Algorithm::Sparse: writeSparse (bw, data); break;
475 case Algorithm::VerySparse: writeVerySparse (bw, data); break;
476 case Algorithm::Incremental: writeIncremental (bw, data); break;
477 case Algorithm::Huffman: writeHuffman (bw, data); break;
478 default: ALIB_ERROR(
"BITBUFFER/AC",
479 "Internal error: Unknown compression algorithm number read") break;
482 auto bufferFill= bw.Usage();
485 statistics->writeTimes [algoNo]+= tm.Age();
486 statistics->sumCompressed[algoNo]+= (bufferFill - initialBufferFill)/8;
490 "Array compression: Nothing written")
491 if( leastBits > bufferFill - initialBufferFill ) {
492 leastBits= bufferFill - initialBufferFill;
498 #if ALIB_DEBUG_ARRAY_COMPRESSION
501 BitReader br(bw.GetBuffer(), initialBufferState);
502 if( multipleAlgorithms ) {
503 auto readBackAlgo=
Algorithm( 1 << br.Read<3>() );
505 "Wrong algorithm id was read back. This must never happen." )
508 data.dbgIsCheckRead=
true;
519 "Internal error: Unknown compression algorithm number read")
524 data.dbgIsCheckRead= false;
528 if( !multipleAlgorithms )
533 statistics->ctdCompressions++;
534 statistics->sumUncompressed+= data.length() *
sizeof(TValue);
535 statistics->ctdWins [bestAlgoNo]++;
536 statistics->sumCompressedWon [bestAlgoNo]+= leastBits/8;
537 statistics->sumUnCompressedWon[bestAlgoNo]+= data.length() *
sizeof(TValue);
542if( multipleAlgorithms && (bestAlgo != lastAlgo) ) {
543 bw.Reset( initialBufferState );
544 bw.Write<3>( bestAlgoNo );
553 default:
ALIB_ERROR(
"BITBUFFER/AC",
"Internal error: Unknown compression "
554 "algorithm number read") break;
560return std::make_pair( leastBits, bestAlgo );
563template <typename TValue>
569 "No algorithms to check given" )
570 bool multipleAlgorithms= CountElements(algorithmsToTry) > 1;
573 auto algo= multipleAlgorithms ?
Algorithm( 1 << br.Read<3>() )
583 default:
ALIB_ERROR(
"BITBUFFER",
"Internal error: Unknown compression "
584 "algorithm number read") break;
590 statistics->readTimes[algoNo]+= tm.Age();
591 statistics->ctdReads [algoNo]++;
Reads bits from a BitBufferBase.
Writes bits into a BitBufferBase.
TUI maxDec
Maximum decrease between two adjacent values.
size_t distance
The distance in memory between two array values.
char * firstVal
Pointer to the first array value.
TUI max
Maximum value (when zig-zag encoded)
void set(size_t idx, TUI value)
TUI maxInc
Maximum increase between two adjacent values.
TUI min
Maximum value (when zig-zag encoded)
TUI minInc
Minimum increase between two adjacent values.
Array(const TIntegral &firstValue, const TIntegral &secondValue, size_t length)
TUI get(size_t idx) const
size_t len
The length of the array.
TUI minDec
Minimum decrease between two adjacent values.
Array(const TIntegral *arrayStart, size_t length)
typename std::make_unsigned< TIntegral >::type TUI
The unsigned version of template type TIntegral.
ArrayCompressor()=delete
Deleted default constructor (this class cannot be created)
static std::pair< size_t, Algorithm > Compress(BitWriter &bitWriter, Array< TValue > &data, Algorithm algorithmsToTry=Algorithm::ALL, Statistics *statistics=nullptr)
static void Decompress(BitReader &bitReader, Array< TValue > &data, Algorithm algorithm=Algorithm::ALL, Statistics *statistics=nullptr)
static constexpr int NumberOfAlgorithms
The number of algorithms implemented.
@ Huffman
Huffman encoding (byte based).
@ Incremental
Only distances of the values are written.
@ ALL
All compression methods selected.
@ Sparse
Writes '1' if next value is equal to previous, '0' plus next value otherwise.
@ MinMax
Stores the differences between the minimum and maximum value found.
@ VerySparse
Writes the number of following equal or non equal values.
@ NONE
No compression method selected.
#define ALIB_ENUMS_MAKE_BITWISE(TEnum)
#define ALIB_WARNINGS_RESTORE
#define ALIB_ENUMS_ASSIGN_RECORD(TEnum, TRecord)
#define ALIB_ASSERT_WARNING(cond, domain,...)
#define ALIB_ENUMS_MAKE_ITERABLE(TEnum, StopElement)
#define ALIB_ERROR(domain,...)
#define ALIB_WARNINGS_IGNORE_DOCS
#define ALIB_WARNINGS_ALLOW_SPARSE_ENUM_SWITCH
#define ALIB_ASSERT_ERROR(cond, domain,...)
void writeHuffman(BitWriter &bw, ArrayCompressor::Array< TI > &data)
void writeVerySparse(BitWriter &bw, ArrayCompressor::Array< TI > &data)
void readHuffman(BitReader &br, ArrayCompressor::Array< TI > &data)
void readSparse(BitReader &br, ArrayCompressor::Array< TI > &data)
void writeUncompressed(BitWriter &bw, ArrayCompressor::Array< TI > &data)
void readMinMax(BitReader &br, ArrayCompressor::Array< TI > &data)
void writeIncremental(BitWriter &bw, ArrayCompressor::Array< TI > &data)
void readVerySparse(BitReader &br, ArrayCompressor::Array< TI > &data)
void readUncompressed(BitReader &br, ArrayCompressor::Array< TI > &data)
void readIncremental(BitReader &br, ArrayCompressor::Array< TI > &data)
void writeSparse(BitWriter &bw, ArrayCompressor::Array< TI > &data)
void writeMinMax(BitWriter &bw, ArrayCompressor::Array< TI > &data)
constexpr std::underlying_type< TEnum >::type ToSequentialEnumeration(TEnum element)
bitbuffer::BitReader BitReader
Type alias in namespace alib.
strings::TAString< character, lang::HeapAllocator > AString
Type alias in namespace alib.
enumops::EnumIterator< TEnum > EnumIterator
Type alias in namespace alib.
strings::TString< character > String
Type alias in namespace alib.
time::Ticks Ticks
Type alias in namespace alib.
bitbuffer::BitWriter BitWriter
Type alias in namespace alib.
int ctdCompressions
The number of executed compressions.
size_t sumUnCompressedWon[NumberOfAlgorithms]
size_t sumCompressed[NumberOfAlgorithms]
For each algorithm, the sum of resulting bytes of all compressions performed.
int ctdWins[NumberOfAlgorithms]
Ticks::Duration readTimes[NumberOfAlgorithms]
The overall decompression time of each algorithm.
size_t sumCompressedWon[NumberOfAlgorithms]
int ctdReads[NumberOfAlgorithms]
The number of measured decompression runs of each algorithm.
Statistics & operator+=(const Statistics &other)
ALIB_DLL void Print(AString &result, const String &headline, bool printTotals)
size_t sumUncompressed
The overall given array data to compress.
Ticks::Duration writeTimes[NumberOfAlgorithms]
The overall compression time of each algorithm.