7zip/Compress/BZip2Decoder.cpp

// BZip2Decoder.cpp

#include "StdAfx.h"

// #include "CopyCoder.h"

/*
#include <stdio.h>
#include "../../../C/CpuTicks.h"
*/
#define TICKS_START
#define TICKS_UPDATE(n)


/*
#define PRIN(s) printf(s "\n"); fflush(stdout);
#define PRIN_VAL(s, val) printf(s " = %u \n", val); fflush(stdout);
*/

#define PRIN(s)
#define PRIN_VAL(s, val)

#define PRIN_MT(s) PRIN("    " s)

#include "../../../C/Alloc.h"

#include "../Common/StreamUtils.h"

#include "BZip2Decoder.h"


namespace NCompress {
namespace NBZip2 {

// #undef NO_INLINE
#define NO_INLINE MY_NO_INLINE

#define BZIP2_BYTE_MODE


static const UInt32 kInBufSize = (UInt32)1 << 17;
static const size_t kOutBufSize = (size_t)1 << 20;

static const UInt32 kProgressStep = (UInt32)1 << 16;


static const UInt16 kRandNums[512] = {
   619, 720, 127, 481, 931, 816, 813, 233, 566, 247,
   985, 724, 205, 454, 863, 491, 741, 242, 949, 214,
   733, 859, 335, 708, 621, 574, 73, 654, 730, 472,
   419, 436, 278, 496, 867, 210, 399, 680, 480, 51,
   878, 465, 811, 169, 869, 675, 611, 697, 867, 561,
   862, 687, 507, 283, 482, 129, 807, 591, 733, 623,
   150, 238, 59, 379, 684, 877, 625, 169, 643, 105,
   170, 607, 520, 932, 727, 476, 693, 425, 174, 647,
   73, 122, 335, 530, 442, 853, 695, 249, 445, 515,
   909, 545, 703, 919, 874, 474, 882, 500, 594, 612,
   641, 801, 220, 162, 819, 984, 589, 513, 495, 799,
   161, 604, 958, 533, 221, 400, 386, 867, 600, 782,
   382, 596, 414, 171, 516, 375, 682, 485, 911, 276,
   98, 553, 163, 354, 666, 933, 424, 341, 533, 870,
   227, 730, 475, 186, 263, 647, 537, 686, 600, 224,
   469, 68, 770, 919, 190, 373, 294, 822, 808, 206,
   184, 943, 795, 384, 383, 461, 404, 758, 839, 887,
   715, 67, 618, 276, 204, 918, 873, 777, 604, 560,
   951, 160, 578, 722, 79, 804, 96, 409, 713, 940,
   652, 934, 970, 447, 318, 353, 859, 672, 112, 785,
   645, 863, 803, 350, 139, 93, 354, 99, 820, 908,
   609, 772, 154, 274, 580, 184, 79, 626, 630, 742,
   653, 282, 762, 623, 680, 81, 927, 626, 789, 125,
   411, 521, 938, 300, 821, 78, 343, 175, 128, 250,
   170, 774, 972, 275, 999, 639, 495, 78, 352, 126,
   857, 956, 358, 619, 580, 124, 737, 594, 701, 612,
   669, 112, 134, 694, 363, 992, 809, 743, 168, 974,
   944, 375, 748, 52, 600, 747, 642, 182, 862, 81,
   344, 805, 988, 739, 511, 655, 814, 334, 249, 515,
   897, 955, 664, 981, 649, 113, 974, 459, 893, 228,
   433, 837, 553, 268, 926, 240, 102, 654, 459, 51,
   686, 754, 806, 760, 493, 403, 415, 394, 687, 700,
   946, 670, 656, 610, 738, 392, 760, 799, 887, 653,
   978, 321, 576, 617, 626, 502, 894, 679, 243, 440,
   680, 879, 194, 572, 640, 724, 926, 56, 204, 700,
   707, 151, 457, 449, 797, 195, 791, 558, 945, 679,
   297, 59, 87, 824, 713, 663, 412, 693, 342, 606,
   134, 108, 571, 364, 631, 212, 174, 643, 304, 329,
   343, 97, 430, 751, 497, 314, 983, 374, 822, 928,
   140, 206, 73, 263, 980, 736, 876, 478, 430, 305,
   170, 514, 364, 692, 829, 82, 855, 953, 676, 246,
   369, 970, 294, 750, 807, 827, 150, 790, 288, 923,
   804, 378, 215, 828, 592, 281, 565, 555, 710, 82,
   896, 831, 547, 261, 524, 462, 293, 465, 502, 56,
   661, 821, 976, 991, 658, 869, 905, 758, 745, 193,
   768, 550, 608, 933, 378, 286, 215, 979, 792, 961,
   61, 688, 793, 644, 986, 403, 106, 366, 905, 644,
   372, 567, 466, 434, 645, 210, 389, 550, 919, 135,
   780, 773, 635, 389, 707, 100, 626, 958, 165, 504,
   920, 176, 193, 713, 857, 265, 203, 50, 668, 108,
   645, 990, 626, 197, 510, 357, 358, 850, 858, 364,
   936, 638
};


enum EState
{
  STATE_STREAM_SIGNATURE,
  STATE_BLOCK_SIGNATURE,

  STATE_BLOCK_START,
  STATE_ORIG_BITS,
  STATE_IN_USE,
  STATE_IN_USE2,
  STATE_NUM_TABLES,
  STATE_NUM_SELECTORS,
  STATE_SELECTORS,
  STATE_LEVELS,

  STATE_BLOCK_SYMBOLS,

  STATE_STREAM_FINISHED
};


#define UPDATE_VAL_2(val) { \
  val |= (UInt32)(*_buf) << (24 - _numBits); \
 _numBits += 8; \
 _buf++; \
}

#define UPDATE_VAL  UPDATE_VAL_2(VAL)

#define READ_BITS(res, num) { \
  while (_numBits < num) { \
    if (_buf == _lim) return SZ_OK; \
    UPDATE_VAL_2(_value) } \
  res = _value >> (32 - num); \
  _value <<= num; \
  _numBits -= num; \
}

#define READ_BITS_8(res, num) { \
  if (_numBits < num) { \
    if (_buf == _lim) return SZ_OK; \
    UPDATE_VAL_2(_value) } \
  res = _value >> (32 - num); \
  _value <<= num; \
  _numBits -= num; \
}

#define READ_BIT(res) READ_BITS_8(res, 1)


#define VAL _value2
#define BLOCK_SIZE blockSize2
#define RUN_COUNTER runCounter2

#define LOAD_LOCAL \
    UInt32 VAL = this->_value; \
    UInt32 BLOCK_SIZE = this->blockSize; \
    UInt32 RUN_COUNTER = this->runCounter; \

#define SAVE_LOCAL \
    this->_value = VAL; \
    this->blockSize = BLOCK_SIZE; \
    this->runCounter = RUN_COUNTER; \


SRes CBitDecoder::ReadByte(int &b)
{
  b = -1;
  READ_BITS_8(b, 8);
  return SZ_OK;
}


NO_INLINE
SRes CBase::ReadStreamSignature2()
{
  for (;;)
  {
    unsigned b;
    READ_BITS_8(b, 8);

    if (   state2 == 0 && b != kArSig0
        || state2 == 1 && b != kArSig1
        || state2 == 2 && b != kArSig2
        || state2 == 3 && (b <= kArSig3 || b > kArSig3 + kBlockSizeMultMax))
      return SZ_ERROR_DATA;
    state2++;

    if (state2 == 4)
    {
      blockSizeMax = (UInt32)(b - kArSig3) * kBlockSizeStep;
      CombinedCrc.Init();
      state = STATE_BLOCK_SIGNATURE;
      state2 = 0;
      return SZ_OK;
    }
  }
}


bool IsEndSig(const Byte *p) throw()
{
  return
    p[0] == kFinSig0 &&
    p[1] == kFinSig1 &&
    p[2] == kFinSig2 &&
    p[3] == kFinSig3 &&
    p[4] == kFinSig4 &&
    p[5] == kFinSig5;
}

bool IsBlockSig(const Byte *p) throw()
{
  return
    p[0] == kBlockSig0 &&
    p[1] == kBlockSig1 &&
    p[2] == kBlockSig2 &&
    p[3] == kBlockSig3 &&
    p[4] == kBlockSig4 &&
    p[5] == kBlockSig5;
}


NO_INLINE
SRes CBase::ReadBlockSignature2()
{
  while (state2 < 10)
  {
    unsigned b;
    READ_BITS_8(b, 8);
    temp[state2] = (Byte)b;
    state2++;
  }

  crc = 0;
  for (unsigned i = 0; i < 4; i++)
  {
    crc <<= 8;
    crc |= temp[6 + i];
  }

  if (IsBlockSig(temp))
  {
    if (!IsBz)
      NumStreams++;
    NumBlocks++;
    IsBz = true;
    CombinedCrc.Update(crc);
    state = STATE_BLOCK_START;
    return SZ_OK;
  }

  if (!IsEndSig(temp))
    return SZ_ERROR_DATA;

  if (!IsBz)
    NumStreams++;
  IsBz = true;

  if (_value != 0)
    MinorError = true;

  AlignToByte();

  state = STATE_STREAM_FINISHED;
  if (crc != CombinedCrc.GetDigest())
  {
    StreamCrcError = true;
    return SZ_ERROR_DATA;
  }
  return SZ_OK;
}


NO_INLINE
SRes CBase::ReadBlock2()
{
  if (state != STATE_BLOCK_SYMBOLS) {
  PRIN("ReadBlock2")

  if (state == STATE_BLOCK_START)
  {
    if (Props.randMode)
    {
      READ_BIT(Props.randMode);
    }
    state = STATE_ORIG_BITS;
    // g_Tick = GetCpuTicks();
  }

  if (state == STATE_ORIG_BITS)
  {
    READ_BITS(Props.origPtr, kNumOrigBits);
    if (Props.origPtr >= blockSizeMax)
      return SZ_ERROR_DATA;
    state = STATE_IN_USE;
  }

  // why original code compares origPtr to (UInt32)(10 + blockSizeMax)) ?

  if (state == STATE_IN_USE)
  {
    READ_BITS(state2, 16);
    state = STATE_IN_USE2;
    state3 = 0;
    numInUse = 0;
    mtf.StartInit();
  }

  if (state == STATE_IN_USE2)
  {
    for (; state3 < 256; state3++)
      if (state2 & ((UInt32)0x8000 >> (state3 >> 4)))
      {
        unsigned b;
        READ_BIT(b);
        if (b)
          mtf.Add(numInUse++, (Byte)state3);
      }
    if (numInUse == 0)
      return SZ_ERROR_DATA;
    state = STATE_NUM_TABLES;
  }


  if (state == STATE_NUM_TABLES)
  {
    READ_BITS_8(numTables, kNumTablesBits);
    state = STATE_NUM_SELECTORS;
    if (numTables < kNumTablesMin || numTables > kNumTablesMax)
      return SZ_ERROR_DATA;
  }

  if (state == STATE_NUM_SELECTORS)
  {
    READ_BITS(numSelectors, kNumSelectorsBits);
    state = STATE_SELECTORS;
    state2 = 0x543210;
    state3 = 0;
    state4 = 0;
    if (numSelectors == 0 || numSelectors > kNumSelectorsMax)
      return SZ_ERROR_DATA;
  }

  if (state == STATE_SELECTORS)
  {
    const unsigned kMtfBits = 4;
    const UInt32 kMtfMask = (1 << kMtfBits) - 1;
    do
    {
      for (;;)
      {
        unsigned b;
        READ_BIT(b);
        if (!b)
          break;
        if (++state4 >= numTables)
          return SZ_ERROR_DATA;
      }
      UInt32 tmp = (state2 >> (kMtfBits * state4)) & kMtfMask;
      UInt32 mask = ((UInt32)1 << ((state4 + 1) * kMtfBits)) - 1;
      state4 = 0;
      state2 = ((state2 << kMtfBits) & mask) | (state2 & ~mask) | tmp;
      selectors[state3] = (Byte)tmp;
    }
    while (++state3 < numSelectors);

    state = STATE_LEVELS;
    state2 = 0;
    state3 = 0;
  }

  if (state == STATE_LEVELS)
  {
    do
    {
      if (state3 == 0)
      {
        READ_BITS_8(state3, kNumLevelsBits);
        state4 = 0;
        state5 = 0;
      }
      const unsigned alphaSize = numInUse + 2;
      for (; state4 < alphaSize; state4++)
      {
        for (;;)
        {
          if (state3 < 1 || state3 > kMaxHuffmanLen)
            return SZ_ERROR_DATA;

          if (state5 == 0)
          {
            unsigned b;
            READ_BIT(b);
            if (!b)
              break;
          }

          state5 = 1;
          unsigned b;
          READ_BIT(b);

          state5 = 0;
          state3++;
          state3 -= (b << 1);
        }
        lens[state4] = (Byte)state3;
        state5 = 0;
      }

      // lbzip2 2.5 can produce dummy tree, where lens[i] = kMaxHuffmanLen
      // BuildFull() returns error for such tree
      /*
      for (unsigned i = state4; i < kMaxAlphaSize; i++)
        lens[i] = 0;
      if (!huffs[state2].Build(lens))
      */
      if (!huffs[state2].BuildFull(lens, state4))
        return SZ_ERROR_DATA;
      state3 = 0;
    }
    while (++state2 < numTables);

    {
      UInt32 *counters = this->Counters;
      for (unsigned i = 0; i < 256; i++)
        counters[i] = 0;
    }

    state = STATE_BLOCK_SYMBOLS;

    groupIndex = 0;
    groupSize = kGroupSize;
    runPower = 0;
    runCounter = 0;
    blockSize = 0;
  }

  if (state != STATE_BLOCK_SYMBOLS)
    return SZ_ERROR_DATA;

  // g_Ticks[3] += GetCpuTicks() - g_Tick;

  }

  {
    LOAD_LOCAL
    const CHuffmanDecoder *huff = &huffs[selectors[groupIndex]];

    for (;;)
    {
      if (groupSize == 0)
      {
        if (++groupIndex >= numSelectors)
          return SZ_ERROR_DATA;
        huff = &huffs[selectors[groupIndex]];
        groupSize = kGroupSize;
      }

      if (_numBits <= 8 &&
          _buf != _lim) { UPDATE_VAL
      if (_buf != _lim) { UPDATE_VAL
      if (_buf != _lim) { UPDATE_VAL }}}

      UInt32 sym;
      UInt32 val = VAL >> (32 - kMaxHuffmanLen);
      if (val >= huff->_limits[kNumTableBits])
      {
        if (_numBits <= kMaxHuffmanLen && _buf != _lim) { UPDATE_VAL
        if (_numBits <= kMaxHuffmanLen && _buf != _lim) { UPDATE_VAL }}

        val = VAL >> (32 - kMaxHuffmanLen);
        unsigned len;
        for (len = kNumTableBits + 1; val >= huff->_limits[len]; len++);
        /*
        if (len > kNumBitsMax)
          return SZ_ERROR_DATA; // that check is required, if NHuffman::Build() was used instead of BuildFull()
        */
        if (_numBits < len)
        {
          SAVE_LOCAL
          return SZ_OK;
        }
        sym = huff->_symbols[huff->_poses[len] + ((val - huff->_limits[(size_t)len - 1]) >> (kNumBitsMax - len))];
        VAL <<= len;
        _numBits -= len;
      }
      else
      {
        sym = huff->_lens[val >> (kMaxHuffmanLen - kNumTableBits)];
        unsigned len = (sym & NHuffman::kPairLenMask);
        sym >>= NHuffman::kNumPairLenBits;
        if (_numBits < len)
        {
          SAVE_LOCAL
          return SZ_OK;
        }
        VAL <<= len;
        _numBits -= len;
      }

      groupSize--;

      if (sym < 2)
      {
        RUN_COUNTER += ((UInt32)(sym + 1) << runPower);
        runPower++;
        if (blockSizeMax - BLOCK_SIZE < RUN_COUNTER)
          return SZ_ERROR_DATA;
        continue;
      }

      UInt32 *counters = this->Counters;
      if (RUN_COUNTER != 0)
      {
        UInt32 b = (UInt32)(mtf.Buf[0] & 0xFF);
        counters[b] += RUN_COUNTER;
        runPower = 0;
        #ifdef BZIP2_BYTE_MODE
          Byte *dest = (Byte *)(&counters[256 + kBlockSizeMax]) + BLOCK_SIZE;
          const Byte *limit = dest + RUN_COUNTER;
          BLOCK_SIZE += RUN_COUNTER;
          RUN_COUNTER = 0;
          do
          {
            dest[0] = (Byte)b;
            dest[1] = (Byte)b;
            dest[2] = (Byte)b;
            dest[3] = (Byte)b;
            dest += 4;
          }
          while (dest < limit);
        #else
          UInt32 *dest = &counters[256 + BLOCK_SIZE];
          const UInt32 *limit = dest + RUN_COUNTER;
          BLOCK_SIZE += RUN_COUNTER;
          RUN_COUNTER = 0;
          do
          {
            dest[0] = b;
            dest[1] = b;
            dest[2] = b;
            dest[3] = b;
            dest += 4;
          }
          while (dest < limit);
        #endif
      }

      sym -= 1;
      if (sym < numInUse)
      {
        if (BLOCK_SIZE >= blockSizeMax)
          return SZ_ERROR_DATA;

        // UInt32 b = (UInt32)mtf.GetAndMove((unsigned)sym);

        const unsigned lim = sym >> MTF_MOVS;
        const unsigned pos = (sym & MTF_MASK) << 3;
        CMtfVar next = mtf.Buf[lim];
        CMtfVar prev = (next >> pos) & 0xFF;

        #ifdef BZIP2_BYTE_MODE
          ((Byte *)(counters + 256 + kBlockSizeMax))[BLOCK_SIZE++] = (Byte)prev;
        #else
          (counters + 256)[BLOCK_SIZE++] = (UInt32)prev;
        #endif
        counters[prev]++;

        CMtfVar *m = mtf.Buf;
        CMtfVar *mLim = m + lim;
        if (lim != 0)
        {
          do
          {
            CMtfVar n0 = *m;
            *m = (n0 << 8) | prev;
            prev = (n0 >> (MTF_MASK << 3));
          }
          while (++m != mLim);
        }

        CMtfVar mask = (((CMtfVar)0x100 << pos) - 1);
        *mLim = (next & ~mask) | (((next << 8) | prev) & mask);
        continue;
      }

      if (sym != numInUse)
        return SZ_ERROR_DATA;
      break;
    }

    // we write additional item that will be read in DecodeBlock1 for prefetching
    #ifdef BZIP2_BYTE_MODE
      ((Byte *)(Counters + 256 + kBlockSizeMax))[BLOCK_SIZE] = 0;
    #else
      (counters + 256)[BLOCK_SIZE] = 0;
    #endif

    SAVE_LOCAL
    Props.blockSize = blockSize;
    state = STATE_BLOCK_SIGNATURE;
    state2 = 0;

    PRIN_VAL("origPtr", Props.origPtr);
    PRIN_VAL("blockSize", Props.blockSize);

    return (Props.origPtr < Props.blockSize) ? SZ_OK : SZ_ERROR_DATA;
  }
}


NO_INLINE
static void DecodeBlock1(UInt32 *counters, UInt32 blockSize)
{
  {
    UInt32 sum = 0;
    for (UInt32 i = 0; i < 256; i++)
    {
      const UInt32 v = counters[i];
      counters[i] = sum;
      sum += v;
    }
  }

  UInt32 *tt = counters + 256;
  // Compute the T^(-1) vector

  // blockSize--;

  #ifdef BZIP2_BYTE_MODE

  unsigned c = ((const Byte *)(tt + kBlockSizeMax))[0];

  for (UInt32 i = 0; i < blockSize; i++)
  {
    unsigned c1 = c;
    const UInt32 pos = counters[c];
    c = ((const Byte *)(tt + kBlockSizeMax))[(size_t)i + 1];
    counters[c1] = pos + 1;
    tt[pos] = (i << 8) | ((const Byte *)(tt + kBlockSizeMax))[pos];
  }

  /*
  // last iteration without next character prefetching
  {
    const UInt32 pos = counters[c];
    counters[c] = pos + 1;
    tt[pos] = (blockSize << 8) | ((const Byte *)(tt + kBlockSizeMax))[pos];
  }
  */

  #else

  unsigned c = (unsigned)(tt[0] & 0xFF);

  for (UInt32 i = 0; i < blockSize; i++)
  {
    unsigned c1 = c;
    const UInt32 pos = counters[c];
    c = (unsigned)(tt[(size_t)i + 1] & 0xFF);
    counters[c1] = pos + 1;
    tt[pos] |= (i << 8);
  }

  /*
  {
    const UInt32 pos = counters[c];
    counters[c] = pos + 1;
    tt[pos] |= (blockSize << 8);
  }
  */

  #endif


  /*
  for (UInt32 i = 0; i < blockSize; i++)
  {
    #ifdef BZIP2_BYTE_MODE
      const unsigned c = ((const Byte *)(tt + kBlockSizeMax))[i];
      const UInt32 pos = counters[c]++;
      tt[pos] = (i << 8) | ((const Byte *)(tt + kBlockSizeMax))[pos];
    #else
      const unsigned c = (unsigned)(tt[i] & 0xFF);
      const UInt32 pos = counters[c]++;
      tt[pos] |= (i << 8);
    #endif
  }
  */
}


void CSpecState::Init(UInt32 origPtr, unsigned randMode) throw()
{
  _tPos = _tt[_tt[origPtr] >> 8];
   _prevByte = (unsigned)(_tPos & 0xFF);
  _reps = 0;
  _randIndex = 0;
  _randToGo = -1;
  if (randMode)
  {
    _randIndex = 1;
    _randToGo = kRandNums[0] - 2;
  }
  _crc.Init();
}


NO_INLINE
Byte * CSpecState::Decode(Byte *data, size_t size) throw()
{
  if (size == 0)
    return data;

  unsigned prevByte = _prevByte;
  int reps = _reps;
  CBZip2Crc crc = _crc;
  const Byte *lim = data + size;

  while (reps > 0)
  {
    reps--;
    *data++ = (Byte)prevByte;
    crc.UpdateByte(prevByte);
    if (data == lim)
      break;
  }

  UInt32 tPos = _tPos;
  UInt32 blockSize = _blockSize;
  const UInt32 *tt = _tt;

  if (data != lim && blockSize)

  for (;;)
  {
    unsigned b = (unsigned)(tPos & 0xFF);
    tPos = tt[tPos >> 8];
    blockSize--;

    if (_randToGo >= 0)
    {
      if (_randToGo == 0)
      {
        b ^= 1;
        _randToGo = kRandNums[_randIndex];
        _randIndex++;
        _randIndex &= 0x1FF;
      }
      _randToGo--;
    }

    if (reps != -(int)kRleModeRepSize)
    {
      if (b != prevByte)
        reps = 0;
      reps--;
      prevByte = b;
      *data++ = (Byte)b;
      crc.UpdateByte(b);
      if (data == lim || blockSize == 0)
        break;
      continue;
    }

    reps = b;
    while (reps)
    {
      reps--;
      *data++ = (Byte)prevByte;
      crc.UpdateByte(prevByte);
      if (data == lim)
        break;
    }
    if (data == lim)
      break;
    if (blockSize == 0)
      break;
  }

  if (blockSize == 1 && reps == -(int)kRleModeRepSize)
  {
    unsigned b = (unsigned)(tPos & 0xFF);
    tPos = tt[tPos >> 8];
    blockSize--;

    if (_randToGo >= 0)
    {
      if (_randToGo == 0)
      {
        b ^= 1;
        _randToGo = kRandNums[_randIndex];
        _randIndex++;
        _randIndex &= 0x1FF;
      }
      _randToGo--;
    }

    reps = b;
  }

  _tPos = tPos;
  _prevByte = prevByte;
  _reps = reps;
  _crc = crc;
  _blockSize = blockSize;

  return data;
}


HRESULT CDecoder::Flush()
{
  if (_writeRes == S_OK)
  {
    _writeRes = WriteStream(_outStream, _outBuf, _outPos);
    _outWritten += _outPos;
    _outPos = 0;
  }
  return _writeRes;
}


NO_INLINE
HRESULT CDecoder::DecodeBlock(const CBlockProps &props)
{
  _calcedBlockCrc = 0;
  _blockFinished = false;

  CSpecState block;

  block._blockSize = props.blockSize;
  block._tt = _counters + 256;

  block.Init(props.origPtr, props.randMode);

  for (;;)
  {
    Byte *data = _outBuf + _outPos;
    size_t size = kOutBufSize - _outPos;

    if (_outSizeDefined)
    {
      const UInt64 rem = _outSize - _outPosTotal;
      if (size >= rem)
      {
        size = (size_t)rem;
        if (size == 0)
          return FinishMode ? S_FALSE : S_OK;
      }
    }

    TICKS_START
    const size_t processed = block.Decode(data, size) - data;
    TICKS_UPDATE(2)

    _outPosTotal += processed;
    _outPos += processed;

    if (processed >= size)
    {
      RINOK(Flush());
    }

    if (block.Finished())
    {
      _blockFinished = true;
      _calcedBlockCrc = block._crc.GetDigest();
      return S_OK;
    }
  }
}


CDecoder::CDecoder():
    _inBuf(NULL),
    _outBuf(NULL),
    _counters(NULL),
    FinishMode(false),
    _outSizeDefined(false)
{
  #ifndef _7ZIP_ST
  MtMode = false;
  NeedWaitScout = false;
  // ScoutRes = S_OK;
  #endif
}


CDecoder::~CDecoder()
{
  PRIN("\n~CDecoder()");

  #ifndef _7ZIP_ST

  if (Thread.IsCreated())
  {
    WaitScout();

    _block.StopScout = true;

    PRIN("\nScoutEvent.Set()");
    ScoutEvent.Set();

    PRIN("\nThread.Wait()()");
    Thread.Wait();
    PRIN("\n after Thread.Wait()()");
    Thread.Close();

    // if (ScoutRes != S_OK) throw ScoutRes;
  }

  #endif

  BigFree(_counters);
  MidFree(_outBuf);
  MidFree(_inBuf);
}


HRESULT CDecoder::ReadInput()
{
  if (Base._buf != Base._lim || _inputFinished || _inputRes != S_OK)
    return _inputRes;

  _inProcessed += (Base._buf - _inBuf);
  Base._buf = _inBuf;
  Base._lim = _inBuf;
  UInt32 size = 0;
  _inputRes = Base.InStream->Read(_inBuf, kInBufSize, &size);
  _inputFinished = (size == 0);
  Base._lim = _inBuf + size;
  return _inputRes;
}


void CDecoder::StartNewStream()
{
  Base.state = STATE_STREAM_SIGNATURE;
  Base.state2 = 0;
  Base.IsBz = false;
}


HRESULT CDecoder::ReadStreamSignature()
{
  for (;;)
  {
    RINOK(ReadInput());
    SRes res = Base.ReadStreamSignature2();
    if (res != SZ_OK)
      return S_FALSE;
    if (Base.state == STATE_BLOCK_SIGNATURE)
      return S_OK;
    if (_inputFinished)
    {
      Base.NeedMoreInput = true;
      return S_FALSE;
    }
  }
}


HRESULT CDecoder::StartRead()
{
  StartNewStream();
  return ReadStreamSignature();
}


HRESULT CDecoder::ReadBlockSignature()
{
  for (;;)
  {
    RINOK(ReadInput());

    SRes res = Base.ReadBlockSignature2();

    if (Base.state == STATE_STREAM_FINISHED)
      Base.FinishedPackSize = GetInputProcessedSize();
    if (res != SZ_OK)
      return S_FALSE;
    if (Base.state != STATE_BLOCK_SIGNATURE)
      return S_OK;
    if (_inputFinished)
    {
      Base.NeedMoreInput = true;
      return S_FALSE;
    }
  }
}


HRESULT CDecoder::ReadBlock()
{
  for (;;)
  {
    RINOK(ReadInput());

    SRes res = Base.ReadBlock2();

    if (res != SZ_OK)
      return S_FALSE;
    if (Base.state == STATE_BLOCK_SIGNATURE)
      return S_OK;
    if (_inputFinished)
    {
      Base.NeedMoreInput = true;
      return S_FALSE;
    }
  }
}


HRESULT CDecoder::DecodeStreams(ICompressProgressInfo *progress)
{
  {
    #ifndef _7ZIP_ST
    _block.StopScout = false;
    #endif
  }

  RINOK(StartRead());

  UInt64 inPrev = 0;
  UInt64 outPrev = 0;

  {
    #ifndef _7ZIP_ST
    CWaitScout_Releaser waitScout_Releaser(this);

    bool useMt = false;
    #endif

    bool wasFinished = false;

    UInt32 crc = 0;
    UInt32 nextCrc = 0;
    HRESULT nextRes = S_OK;

    UInt64 packPos = 0;

    CBlockProps props;

    props.blockSize = 0;

    for (;;)
    {
      if (progress)
      {
        const UInt64 outCur = GetOutProcessedSize();
        if (packPos - inPrev >= kProgressStep || outCur - outPrev >= kProgressStep)
        {
          RINOK(progress->SetRatioInfo(&packPos, &outCur));
          inPrev = packPos;
          outPrev = outCur;
        }
      }

      if (props.blockSize == 0)
        if (wasFinished || nextRes != S_OK)
          return nextRes;

      if (
          #ifndef _7ZIP_ST
          !useMt &&
          #endif
          !wasFinished && Base.state == STATE_BLOCK_SIGNATURE)
      {
        nextRes = ReadBlockSignature();
        nextCrc = Base.crc;
        packPos = GetInputProcessedSize();

        wasFinished = true;

        if (nextRes != S_OK)
          continue;

        if (Base.state == STATE_STREAM_FINISHED)
        {
          if (!Base.DecodeAllStreams)
          {
            wasFinished = true;
            continue;
          }

          nextRes = StartRead();

          if (Base.NeedMoreInput)
          {
            if (Base.state2 == 0)
              Base.NeedMoreInput = false;
            wasFinished = true;
            nextRes = S_OK;
            continue;
          }

          if (nextRes != S_OK)
            continue;

          wasFinished = false;
          continue;
        }

        wasFinished = false;

        #ifndef _7ZIP_ST
        if (MtMode)
        if (props.blockSize != 0)
        {
          // we start multithreading, if next block is big enough.
          const UInt32 k_Mt_BlockSize_Threshold = (1 << 12);  // (1 << 13)
          if (props.blockSize > k_Mt_BlockSize_Threshold)
          {
            if (!Thread.IsCreated())
            {
              PRIN("=== MT_MODE");
              RINOK(CreateThread());
            }
            useMt = true;
          }
        }
        #endif
      }

      if (props.blockSize == 0)
      {
        crc = nextCrc;

        #ifndef _7ZIP_ST
        if (useMt)
        {
          PRIN("DecoderEvent.Lock()");
          RINOK(DecoderEvent.Lock());
          NeedWaitScout = false;
          PRIN("-- DecoderEvent.Lock()");
          props = _block.Props;
          nextCrc = _block.NextCrc;
          if (_block.Crc_Defined)
            crc = _block.Crc;
          packPos = _block.PackPos;
          wasFinished = _block.WasFinished;
          RINOK(_block.Res);
        }
        else
        #endif
        {
          if (Base.state != STATE_BLOCK_START)
            return E_FAIL;

          TICKS_START
          Base.Props.randMode = 1;
          RINOK(ReadBlock());
          TICKS_UPDATE(0)

          props = Base.Props;
          continue;
        }
      }

      if (props.blockSize != 0)
      {
        TICKS_START
        DecodeBlock1(_counters, props.blockSize);
        TICKS_UPDATE(1)
      }

      #ifndef _7ZIP_ST
      if (useMt && !wasFinished)
      {
        /*
        if (props.blockSize == 0)
        {
          // this codes switches back to single-threadMode
          useMt = false;
          PRIN("=== ST_MODE");
          continue;
          }
        */

        PRIN("ScoutEvent.Set()");
        RINOK(ScoutEvent.Set());
        NeedWaitScout = true;
      }
      #endif

      if (props.blockSize == 0)
        continue;

      RINOK(DecodeBlock(props));

      if (!_blockFinished)
        return nextRes;

      props.blockSize = 0;
      if (_calcedBlockCrc != crc)
      {
        BlockCrcError = true;
        return S_FALSE;
      }
    }
  }
}


bool CDecoder::CreateInputBufer()
{
  if (!_inBuf)
  {
    _inBuf = (Byte *)MidAlloc(kInBufSize);
    if (!_inBuf)
      return false;
  }
  if (!_counters)
  {
    _counters = (UInt32 *)::BigAlloc((256 + kBlockSizeMax) * sizeof(UInt32)
      #ifdef BZIP2_BYTE_MODE
        + kBlockSizeMax
      #endif
        + 256);
    if (!_counters)
      return false;
    Base.Counters = _counters;
  }
  return true;
}


void CDecoder::InitOutSize(const UInt64 *outSize)
{
  _outPosTotal = 0;

  _outSizeDefined = false;
  _outSize = 0;
  if (outSize)
  {
    _outSize = *outSize;
    _outSizeDefined = true;
  }

  BlockCrcError = false;

  Base.InitNumStreams2();
}


STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream,
    const UInt64 * /* inSize */, const UInt64 *outSize, ICompressProgressInfo *progress)
{
  /*
  {
    RINOK(SetInStream(inStream));
    RINOK(SetOutStreamSize(outSize));

    RINOK(CopyStream(this, outStream, progress));
    return ReleaseInStream();
  }
  */

  InitOutSize(outSize);

  _inputFinished = false;
  _inputRes = S_OK;
  _writeRes = S_OK;

  try {

  if (!CreateInputBufer())
    return E_OUTOFMEMORY;

  if (!_outBuf)
  {
    _outBuf = (Byte *)MidAlloc(kOutBufSize);
    if (!_outBuf)
      return E_OUTOFMEMORY;
  }

  Base.InStream = inStream;

  InitInputBuffer();

  _outStream = outStream;
  _outWritten = 0;
  _outPos = 0;

  HRESULT res = DecodeStreams(progress);

  Flush();

  Base.InStream = NULL;
  _outStream = NULL;

  /*
  if (res == S_OK)
    if (FinishMode && inSize && *inSize != GetInputProcessedSize())
      res = S_FALSE;
  */

  if (res != S_OK)
    return res;

  } catch(...) { return E_FAIL; }

  return _writeRes;
}


STDMETHODIMP CDecoder::SetFinishMode(UInt32 finishMode)
{
  FinishMode = (finishMode != 0);
  return S_OK;
}


STDMETHODIMP CDecoder::GetInStreamProcessedSize(UInt64 *value)
{
  *value = GetInputProcessedSize();
  return S_OK;
}


#ifndef _7ZIP_ST

#define RINOK_THREAD(x) { WRes __result_ = (x); if (__result_ != 0) return __result_; }

static THREAD_FUNC_DECL RunScout2(void *p) { ((CDecoder *)p)->RunScout(); return 0; }

HRESULT CDecoder::CreateThread()
{
  RINOK_THREAD(DecoderEvent.CreateIfNotCreated());
  RINOK_THREAD(ScoutEvent.CreateIfNotCreated());
  RINOK_THREAD(Thread.Create(RunScout2, this));
  return S_OK;
}

void CDecoder::RunScout()
{
  for (;;)
  {
    {
      PRIN_MT("ScoutEvent.Lock()");
      WRes wres = ScoutEvent.Lock();
      PRIN_MT("-- ScoutEvent.Lock()");
      if (wres != 0)
      {
        // ScoutRes = wres;
        return;
      }
    }

    CBlock &block = _block;

    if (block.StopScout)
    {
      // ScoutRes = S_OK;
      return;
    }

    block.Res = S_OK;
    block.WasFinished = false;

    HRESULT res = S_OK;

    try
    {
      UInt64 packPos = GetInputProcessedSize();

      block.Props.blockSize = 0;
      block.Crc_Defined = false;
      // block.NextCrc_Defined = false;
      block.NextCrc = 0;

      for (;;)
      {
        if (Base.state == STATE_BLOCK_SIGNATURE)
        {
          res = ReadBlockSignature();

          if (res != S_OK)
            break;

          if (block.Props.blockSize == 0)
          {
            block.Crc = Base.crc;
            block.Crc_Defined = true;
          }
          else
          {
            block.NextCrc = Base.crc;
            // block.NextCrc_Defined = true;
          }

          continue;
        }

        if (Base.state == STATE_BLOCK_START)
        {
          if (block.Props.blockSize != 0)
            break;

          Base.Props.randMode = 1;

          res = ReadBlock();

          PRIN_MT("-- Base.ReadBlock");
          if (res != S_OK)
            break;
          block.Props = Base.Props;
          continue;
        }

        if (Base.state == STATE_STREAM_FINISHED)
        {
          if (!Base.DecodeAllStreams)
          {
            block.WasFinished = true;
            break;
          }

          res = StartRead();

          if (Base.NeedMoreInput)
          {
            if (Base.state2 == 0)
              Base.NeedMoreInput = false;
            block.WasFinished = true;
            res = S_OK;
            break;
          }

          if (res != S_OK)
            break;

          if (GetInputProcessedSize() - packPos > 0) // kProgressStep
            break;
          continue;
        }

        // throw 1;
        res = E_FAIL;
        break;
      }
    }

    catch (...) { res = E_FAIL; }

    if (res != S_OK)
    {
      PRIN_MT("error");
      block.Res = res;
      block.WasFinished = true;
    }

    block.PackPos = GetInputProcessedSize();
    PRIN_MT("DecoderEvent.Set()");
    WRes wres = DecoderEvent.Set();
    if (wres != 0)
    {
      // ScoutRes = wres;
      return;
    }
  }
}


STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads)
{
  MtMode = (numThreads > 1);

  #ifndef BZIP2_BYTE_MODE
  MtMode = false;
  #endif

  // MtMode = false;
  return S_OK;
}

#endif


#ifndef NO_READ_FROM_CODER


STDMETHODIMP CDecoder::SetInStream(ISequentialInStream *inStream)
{
  Base.InStreamRef = inStream;
  Base.InStream = inStream;
  return S_OK;
}


STDMETHODIMP CDecoder::ReleaseInStream()
{
  Base.InStreamRef.Release();
  Base.InStream = NULL;
  return S_OK;
}


STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 *outSize)
{
  InitOutSize(outSize);

  if (!CreateInputBufer())
    return E_OUTOFMEMORY;

  InitInputBuffer();

  StartNewStream();

  _blockFinished = true;

  ErrorResult = S_OK;

  _inputFinished = false;
  _inputRes = S_OK;

  return S_OK;
}


STDMETHODIMP CDecoder::Read(void *data, UInt32 size, UInt32 *processedSize)
{
  *processedSize = 0;

  try {

  if (ErrorResult != S_OK)
    return ErrorResult;

  for (;;)
  {
    if (Base.state == STATE_STREAM_FINISHED)
    {
      if (!Base.DecodeAllStreams)
        return ErrorResult;
      StartNewStream();
      continue;
    }

    if (Base.state == STATE_STREAM_SIGNATURE)
    {
      ErrorResult = ReadStreamSignature();

      if (Base.NeedMoreInput)
        if (Base.state2 == 0 && Base.NumStreams != 0)
        {
          Base.NeedMoreInput = false;
          ErrorResult = S_OK;
          return S_OK;
        }
      if (ErrorResult != S_OK)
        return ErrorResult;
      continue;
    }

    if (_blockFinished && Base.state == STATE_BLOCK_SIGNATURE)
    {
      ErrorResult = ReadBlockSignature();

      if (ErrorResult != S_OK)
        return ErrorResult;

      continue;
    }

    if (_outSizeDefined)
    {
      const UInt64 rem = _outSize - _outPosTotal;
      if (size >= rem)
        size = (UInt32)rem;
    }
    if (size == 0)
      return S_OK;

    if (_blockFinished)
    {
      if (Base.state != STATE_BLOCK_START)
      {
        ErrorResult = E_FAIL;
        return ErrorResult;
      }

      Base.Props.randMode = 1;
      ErrorResult = ReadBlock();

      if (ErrorResult != S_OK)
        return ErrorResult;

      DecodeBlock1(_counters, Base.Props.blockSize);

      _spec._blockSize = Base.Props.blockSize;
      _spec._tt = _counters + 256;
      _spec.Init(Base.Props.origPtr, Base.Props.randMode);

      _blockFinished = false;
    }

    {
      Byte *ptr = _spec.Decode((Byte *)data, size);

      const UInt32 processed = (UInt32)(ptr - (Byte *)data);
      data = ptr;
      size -= processed;
      (*processedSize) += processed;
      _outPosTotal += processed;

      if (_spec.Finished())
      {
        _blockFinished = true;
        if (Base.crc != _spec._crc.GetDigest())
        {
          BlockCrcError = true;
          ErrorResult = S_FALSE;
          return ErrorResult;
        }
      }
    }
  }

  } catch(...) { ErrorResult = S_FALSE; return S_FALSE; }
}


// ---------- NSIS ----------

STDMETHODIMP CNsisDecoder::Read(void *data, UInt32 size, UInt32 *processedSize)
{
  *processedSize = 0;

  try {

  if (ErrorResult != S_OK)
    return ErrorResult;

  if (Base.state == STATE_STREAM_FINISHED)
    return S_OK;

  if (Base.state == STATE_STREAM_SIGNATURE)
  {
    Base.blockSizeMax = 9 * kBlockSizeStep;
    Base.state = STATE_BLOCK_SIGNATURE;
    // Base.state2 = 0;
  }

  for (;;)
  {
    if (_blockFinished && Base.state == STATE_BLOCK_SIGNATURE)
    {
      ErrorResult = ReadInput();
      if (ErrorResult != S_OK)
        return ErrorResult;

      int b;
      Base.ReadByte(b);
      if (b < 0)
      {
        ErrorResult = S_FALSE;
        return ErrorResult;
      }

      if (b == kFinSig0)
      {
        /*
        if (!Base.AreRemainByteBitsEmpty())
          ErrorResult = S_FALSE;
        */
        Base.state = STATE_STREAM_FINISHED;
        return ErrorResult;
      }

      if (b != kBlockSig0)
      {
        ErrorResult = S_FALSE;
        return ErrorResult;
      }

      Base.state = STATE_BLOCK_START;
    }

    if (_outSizeDefined)
    {
      const UInt64 rem = _outSize - _outPosTotal;
      if (size >= rem)
        size = (UInt32)rem;
    }
    if (size == 0)
      return S_OK;

    if (_blockFinished)
    {
      if (Base.state != STATE_BLOCK_START)
      {
        ErrorResult = E_FAIL;
        return ErrorResult;
      }

      Base.Props.randMode = 0;
      ErrorResult = ReadBlock();

      if (ErrorResult != S_OK)
        return ErrorResult;

      DecodeBlock1(_counters, Base.Props.blockSize);

      _spec._blockSize = Base.Props.blockSize;
      _spec._tt = _counters + 256;
      _spec.Init(Base.Props.origPtr, Base.Props.randMode);

      _blockFinished = false;
    }

    {
      Byte *ptr = _spec.Decode((Byte *)data, size);

      const UInt32 processed = (UInt32)(ptr - (Byte *)data);
      data = ptr;
      size -= processed;
      (*processedSize) += processed;
      _outPosTotal += processed;

      if (_spec.Finished())
        _blockFinished = true;
    }
  }

  } catch(...) { ErrorResult = S_FALSE; return S_FALSE; }
}

#endif

}}