Coverage Report

Created: 2026-02-26 07:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/minizip-ng/third_party/ppmd/C/Ppmd8.c
Line
Count
Source
1
/* Ppmd8.c -- PPMdI codec
2
2023-09-07 : Igor Pavlov : Public domain
3
This code is based on PPMd var.I (2002): Dmitry Shkarin : Public domain */
4
5
#include "Precomp.h"
6
7
#include <string.h>
8
9
#include "Ppmd8.h"
10
11
12
13
14
MY_ALIGN(16)
15
static const Byte PPMD8_kExpEscape[16] = { 25, 14, 9, 7, 5, 5, 4, 4, 4, 3, 3, 3, 2, 2, 2, 2 };
16
MY_ALIGN(16)
17
static const UInt16 PPMD8_kInitBinEsc[] = { 0x3CDD, 0x1F3F, 0x59BF, 0x48F3, 0x64A1, 0x5ABC, 0x6632, 0x6051};
18
19
283M
#define MAX_FREQ 124
20
82.9M
#define UNIT_SIZE 12
21
22
39.2M
#define U2B(nu) ((UInt32)(nu) * UNIT_SIZE)
23
74.8M
#define U2I(nu) (p->Units2Indx[(size_t)(nu) - 1])
24
63.1M
#define I2U(indx) ((unsigned)p->Indx2Units[indx])
25
26
27
314M
#define REF(ptr) Ppmd_GetRef(p, ptr)
28
29
22.6M
#define STATS_REF(ptr) ((CPpmd_State_Ref)REF(ptr))
30
31
416M
#define CTX(ref) ((CPpmd8_Context *)Ppmd8_GetContext(p, ref))
32
210M
#define STATS(ctx) Ppmd8_GetStats(p, ctx)
33
73.3M
#define ONE_STATE(ctx) Ppmd8Context_OneState(ctx)
34
267M
#define SUFFIX(ctx) CTX((ctx)->Suffix)
35
36
typedef CPpmd8_Context * PPMD8_CTX_PTR;
37
38
struct CPpmd8_Node_;
39
40
typedef Ppmd_Ref_Type(struct CPpmd8_Node_) CPpmd8_Node_Ref;
41
42
typedef struct CPpmd8_Node_
43
{
44
  UInt32 Stamp;
45
  
46
  CPpmd8_Node_Ref Next;
47
  UInt32 NU;
48
} CPpmd8_Node;
49
50
34.8M
#define NODE(r)  Ppmd_GetPtr_Type(p, r, CPpmd8_Node)
51
52
void Ppmd8_Construct(CPpmd8 *p)
53
11.3k
{
54
11.3k
  unsigned i, k, m;
55
56
11.3k
  p->Base = NULL;
57
58
442k
  for (i = 0, k = 0; i < PPMD_NUM_INDEXES; i++)
59
431k
  {
60
431k
    unsigned step = (i >= 12 ? 4 : (i >> 2) + 1);
61
1.45M
    do { p->Units2Indx[k++] = (Byte)i; } while (--step);
62
431k
    p->Indx2Units[i] = (Byte)k;
63
431k
  }
64
65
11.3k
  p->NS2BSIndx[0] = (0 << 1);
66
11.3k
  p->NS2BSIndx[1] = (1 << 1);
67
11.3k
  memset(p->NS2BSIndx + 2, (2 << 1), 9);
68
11.3k
  memset(p->NS2BSIndx + 11, (3 << 1), 256 - 11);
69
70
68.0k
  for (i = 0; i < 5; i++)
71
56.7k
    p->NS2Indx[i] = (Byte)i;
72
  
73
2.90M
  for (m = i, k = 1; i < 260; i++)
74
2.89M
  {
75
2.89M
    p->NS2Indx[i] = (Byte)m;
76
2.89M
    if (--k == 0)
77
249k
      k = (++m) - 4;
78
2.89M
  }
79
80
11.3k
  memcpy(p->ExpEscape, PPMD8_kExpEscape, 16);
81
11.3k
}
82
83
84
void Ppmd8_Free(CPpmd8 *p, ISzAllocPtr alloc)
85
22.5k
{
86
22.5k
  ISzAlloc_Free(alloc, p->Base);
87
22.5k
  p->Size = 0;
88
22.5k
  p->Base = NULL;
89
22.5k
}
90
91
92
BoolInt Ppmd8_Alloc(CPpmd8 *p, UInt32 size, ISzAllocPtr alloc)
93
11.2k
{
94
11.2k
  if (!p->Base || p->Size != size)
95
11.2k
  {
96
11.2k
    Ppmd8_Free(p, alloc);
97
11.2k
    p->AlignOffset = (4 - size) & 3;
98
11.2k
    if ((p->Base = (Byte *)ISzAlloc_Alloc(alloc, p->AlignOffset + size)) == NULL)
99
0
      return False;
100
11.2k
    p->Size = size;
101
11.2k
  }
102
11.2k
  return True;
103
11.2k
}
104
105
106
107
// ---------- Internal Memory Allocator ----------
108
109
110
111
112
113
114
37.3M
#define EMPTY_NODE 0xFFFFFFFF
115
116
117
static void Ppmd8_InsertNode(CPpmd8 *p, void *node, unsigned indx)
118
35.1M
{
119
35.1M
  ((CPpmd8_Node *)node)->Stamp = EMPTY_NODE;
120
35.1M
  ((CPpmd8_Node *)node)->Next = (CPpmd8_Node_Ref)p->FreeList[indx];
121
35.1M
  ((CPpmd8_Node *)node)->NU = I2U(indx);
122
35.1M
  p->FreeList[indx] = REF(node);
123
35.1M
  p->Stamps[indx]++;
124
35.1M
}
125
126
127
static void *Ppmd8_RemoveNode(CPpmd8 *p, unsigned indx)
128
31.3M
{
129
31.3M
  CPpmd8_Node *node = NODE((CPpmd8_Node_Ref)p->FreeList[indx]);
130
31.3M
  p->FreeList[indx] = node->Next;
131
31.3M
  p->Stamps[indx]--;
132
133
31.3M
  return node;
134
31.3M
}
135
136
137
static void Ppmd8_SplitBlock(CPpmd8 *p, void *ptr, unsigned oldIndx, unsigned newIndx)
138
7.78M
{
139
7.78M
  unsigned i, nu = I2U(oldIndx) - I2U(newIndx);
140
7.78M
  ptr = (Byte *)ptr + U2B(I2U(newIndx));
141
7.78M
  if (I2U(i = U2I(nu)) != nu)
142
3.23M
  {
143
3.23M
    unsigned k = I2U(--i);
144
3.23M
    Ppmd8_InsertNode(p, ((Byte *)ptr) + U2B(k), nu - k - 1);
145
3.23M
  }
146
7.78M
  Ppmd8_InsertNode(p, ptr, i);
147
7.78M
}
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
static void Ppmd8_GlueFreeBlocks(CPpmd8 *p)
163
853
{
164
  /*
165
  we use first UInt32 field of 12-bytes UNITs as record type stamp
166
    CPpmd_State    { Byte Symbol; Byte Freq; : Freq != 0xFF
167
    CPpmd8_Context { Byte NumStats; Byte Flags; UInt16 SummFreq;  : Flags != 0xFF ???
168
    CPpmd8_Node    { UInt32 Stamp            : Stamp == 0xFFFFFFFF for free record
169
                                             : Stamp == 0 for guard
170
    Last 12-bytes UNIT in array is always contains 12-bytes order-0 CPpmd8_Context record
171
  */
172
853
  CPpmd8_Node_Ref n;
173
174
853
  p->GlueCount = 1 << 13;
175
853
  memset(p->Stamps, 0, sizeof(p->Stamps));
176
  
177
  /* we set guard NODE at LoUnit */
178
853
  if (p->LoUnit != p->HiUnit)
179
511
    ((CPpmd8_Node *)(void *)p->LoUnit)->Stamp = 0;
180
181
853
  {
182
    /* Glue free blocks */
183
853
    CPpmd8_Node_Ref *prev = &n;
184
853
    unsigned i;
185
33.2k
    for (i = 0; i < PPMD_NUM_INDEXES; i++)
186
32.4k
    {
187
188
32.4k
      CPpmd8_Node_Ref next = (CPpmd8_Node_Ref)p->FreeList[i];
189
32.4k
      p->FreeList[i] = 0;
190
2.01M
      while (next != 0)
191
1.98M
      {
192
1.98M
        CPpmd8_Node *node = NODE(next);
193
1.98M
        UInt32 nu = node->NU;
194
1.98M
        *prev = next;
195
1.98M
        next = node->Next;
196
1.98M
        if (nu != 0)
197
1.52M
        {
198
1.52M
          CPpmd8_Node *node2;
199
1.52M
          prev = &(node->Next);
200
2.16M
          while ((node2 = node + nu)->Stamp == EMPTY_NODE)
201
639k
          {
202
639k
            nu += node2->NU;
203
639k
            node2->NU = 0;
204
639k
            node->NU = nu;
205
639k
          }
206
1.52M
        }
207
1.98M
      }
208
32.4k
    }
209
210
853
    *prev = 0;
211
853
  }
212
213
214
215
  
216
217
218
219
220
221
222
  
223
224
  
225
  
226
  
227
  
228
  
229
  
230
  
231
  
232
  /* Fill lists of free blocks */
233
1.52M
  while (n != 0)
234
1.52M
  {
235
1.52M
    CPpmd8_Node *node = NODE(n);
236
1.52M
    UInt32 nu = node->NU;
237
1.52M
    unsigned i;
238
1.52M
    n = node->Next;
239
1.52M
    if (nu == 0)
240
179k
      continue;
241
1.37M
    for (; nu > 128; nu -= 128, node += 128)
242
34.0k
      Ppmd8_InsertNode(p, node, PPMD_NUM_INDEXES - 1);
243
1.34M
    if (I2U(i = U2I(nu)) != nu)
244
92.0k
    {
245
92.0k
      unsigned k = I2U(--i);
246
92.0k
      Ppmd8_InsertNode(p, node + k, (unsigned)nu - k - 1);
247
92.0k
    }
248
1.34M
    Ppmd8_InsertNode(p, node, i);
249
1.34M
  }
250
853
}
251
252
253
Z7_NO_INLINE
254
static void *Ppmd8_AllocUnitsRare(CPpmd8 *p, unsigned indx)
255
8.78M
{
256
8.78M
  unsigned i;
257
  
258
8.78M
  if (p->GlueCount == 0)
259
853
  {
260
853
    Ppmd8_GlueFreeBlocks(p);
261
853
    if (p->FreeList[indx] != 0)
262
741
      return Ppmd8_RemoveNode(p, indx);
263
853
  }
264
  
265
8.78M
  i = indx;
266
  
267
8.78M
  do
268
80.7M
  {
269
80.7M
    if (++i == PPMD_NUM_INDEXES)
270
1.00M
    {
271
1.00M
      UInt32 numBytes = U2B(I2U(indx));
272
1.00M
      Byte *us = p->UnitsStart;
273
1.00M
      p->GlueCount--;
274
1.00M
      return ((UInt32)(us - p->Text) > numBytes) ? (p->UnitsStart = us - numBytes) : (NULL);
275
1.00M
    }
276
80.7M
  }
277
79.7M
  while (p->FreeList[i] == 0);
278
  
279
7.77M
  {
280
7.77M
    void *block = Ppmd8_RemoveNode(p, i);
281
7.77M
    Ppmd8_SplitBlock(p, block, i, indx);
282
7.77M
    return block;
283
8.78M
  }
284
8.78M
}
285
286
287
static void *Ppmd8_AllocUnits(CPpmd8 *p, unsigned indx)
288
46.9M
{
289
46.9M
  if (p->FreeList[indx] != 0)
290
19.7M
    return Ppmd8_RemoveNode(p, indx);
291
27.2M
  {
292
27.2M
    UInt32 numBytes = U2B(I2U(indx));
293
27.2M
    Byte *lo = p->LoUnit;
294
27.2M
    if ((UInt32)(p->HiUnit - lo) >= numBytes)
295
23.5M
    {
296
23.5M
      p->LoUnit = lo + numBytes;
297
23.5M
      return lo;
298
23.5M
    }
299
27.2M
  }
300
3.63M
  return Ppmd8_AllocUnitsRare(p, indx);
301
27.2M
}
302
303
304
#define MEM_12_CPY(dest, src, num) \
305
22.6M
  { UInt32 *d = (UInt32 *)(dest); \
306
22.6M
    const UInt32 *z = (const UInt32 *)(src); \
307
22.6M
    unsigned n = (num); \
308
267M
    do { \
309
267M
      d[0] = z[0]; \
310
267M
      d[1] = z[1]; \
311
267M
      d[2] = z[2]; \
312
267M
      z += 3; \
313
267M
      d += 3; \
314
267M
    } while (--n); \
315
22.6M
  }
316
317
318
319
static void *ShrinkUnits(CPpmd8 *p, void *oldPtr, unsigned oldNU, unsigned newNU)
320
35.0k
{
321
35.0k
  unsigned i0 = U2I(oldNU);
322
35.0k
  unsigned i1 = U2I(newNU);
323
35.0k
  if (i0 == i1)
324
3.18k
    return oldPtr;
325
31.9k
  if (p->FreeList[i1] != 0)
326
24.3k
  {
327
24.3k
    void *ptr = Ppmd8_RemoveNode(p, i1);
328
24.3k
    MEM_12_CPY(ptr, oldPtr, newNU)
329
24.3k
    Ppmd8_InsertNode(p, oldPtr, i0);
330
24.3k
    return ptr;
331
24.3k
  }
332
7.55k
  Ppmd8_SplitBlock(p, oldPtr, i0, i1);
333
7.55k
  return oldPtr;
334
31.9k
}
335
336
337
static void FreeUnits(CPpmd8 *p, void *ptr, unsigned nu)
338
0
{
339
0
  Ppmd8_InsertNode(p, ptr, U2I(nu));
340
0
}
341
342
343
static void SpecialFreeUnit(CPpmd8 *p, void *ptr)
344
150
{
345
150
  if ((Byte *)ptr != p->UnitsStart)
346
148
    Ppmd8_InsertNode(p, ptr, 0);
347
2
  else
348
2
  {
349
    #ifdef PPMD8_FREEZE_SUPPORT
350
    *(UInt32 *)ptr = EMPTY_NODE; /* it's used for (Flags == 0xFF) check in RemoveBinContexts() */
351
    #endif
352
2
    p->UnitsStart += UNIT_SIZE;
353
2
  }
354
150
}
355
356
357
/*
358
static void *MoveUnitsUp(CPpmd8 *p, void *oldPtr, unsigned nu)
359
{
360
  unsigned indx = U2I(nu);
361
  void *ptr;
362
  if ((Byte *)oldPtr > p->UnitsStart + (1 << 14) || REF(oldPtr) > p->FreeList[indx])
363
    return oldPtr;
364
  ptr = Ppmd8_RemoveNode(p, indx);
365
  MEM_12_CPY(ptr, oldPtr, nu)
366
  if ((Byte *)oldPtr != p->UnitsStart)
367
    Ppmd8_InsertNode(p, oldPtr, indx);
368
  else
369
    p->UnitsStart += U2B(I2U(indx));
370
  return ptr;
371
}
372
*/
373
374
static void ExpandTextArea(CPpmd8 *p)
375
0
{
376
0
  UInt32 count[PPMD_NUM_INDEXES];
377
0
  unsigned i;
378
 
379
0
  memset(count, 0, sizeof(count));
380
0
  if (p->LoUnit != p->HiUnit)
381
0
    ((CPpmd8_Node *)(void *)p->LoUnit)->Stamp = 0;
382
  
383
0
  {
384
0
    CPpmd8_Node *node = (CPpmd8_Node *)(void *)p->UnitsStart;
385
0
    while (node->Stamp == EMPTY_NODE)
386
0
    {
387
0
      UInt32 nu = node->NU;
388
0
      node->Stamp = 0;
389
0
      count[U2I(nu)]++;
390
0
      node += nu;
391
0
    }
392
0
    p->UnitsStart = (Byte *)node;
393
0
  }
394
  
395
0
  for (i = 0; i < PPMD_NUM_INDEXES; i++)
396
0
  {
397
0
    UInt32 cnt = count[i];
398
0
    if (cnt == 0)
399
0
      continue;
400
0
    {
401
0
      CPpmd8_Node_Ref *prev = (CPpmd8_Node_Ref *)&p->FreeList[i];
402
0
      CPpmd8_Node_Ref n = *prev;
403
0
      p->Stamps[i] -= cnt;
404
0
      for (;;)
405
0
      {
406
0
        CPpmd8_Node *node = NODE(n);
407
0
        n = node->Next;
408
0
        if (node->Stamp != 0)
409
0
        {
410
0
          prev = &node->Next;
411
0
          continue;
412
0
        }
413
0
        *prev = n;
414
0
        if (--cnt == 0)
415
0
          break;
416
0
      }
417
0
    }
418
0
  }
419
0
}
420
421
422
197M
#define SUCCESSOR(p) Ppmd_GET_SUCCESSOR(p)
423
static void Ppmd8State_SetSuccessor(CPpmd_State *p, CPpmd_Void_Ref v)
424
210M
{
425
210M
  Ppmd_SET_SUCCESSOR(p, v)
426
210M
}
427
428
12.2k
#define RESET_TEXT(offs) { p->Text = p->Base + p->AlignOffset + (offs); }
429
430
Z7_NO_INLINE
431
static
432
void Ppmd8_RestartModel(CPpmd8 *p)
433
11.7k
{
434
11.7k
  unsigned i, k, m;
435
436
11.7k
  memset(p->FreeList, 0, sizeof(p->FreeList));
437
11.7k
  memset(p->Stamps, 0, sizeof(p->Stamps));
438
11.7k
  RESET_TEXT(0)
439
11.7k
  p->HiUnit = p->Text + p->Size;
440
11.7k
  p->LoUnit = p->UnitsStart = p->HiUnit - p->Size / 8 / UNIT_SIZE * 7 * UNIT_SIZE;
441
11.7k
  p->GlueCount = 0;
442
443
11.7k
  p->OrderFall = p->MaxOrder;
444
11.7k
  p->RunLength = p->InitRL = -(Int32)((p->MaxOrder < 12) ? p->MaxOrder : 12) - 1;
445
11.7k
  p->PrevSuccess = 0;
446
447
11.7k
  {
448
11.7k
    CPpmd8_Context *mc = (PPMD8_CTX_PTR)(void *)(p->HiUnit -= UNIT_SIZE); /* AllocContext(p); */
449
11.7k
    CPpmd_State *s = (CPpmd_State *)p->LoUnit; /* Ppmd8_AllocUnits(p, PPMD_NUM_INDEXES - 1); */
450
    
451
11.7k
    p->LoUnit += U2B(256 / 2);
452
11.7k
    p->MaxContext = p->MinContext = mc;
453
11.7k
    p->FoundState = s;
454
11.7k
    mc->Flags = 0;
455
11.7k
    mc->NumStats = 256 - 1;
456
11.7k
    mc->Union2.SummFreq = 256 + 1;
457
11.7k
    mc->Union4.Stats = REF(s);
458
11.7k
    mc->Suffix = 0;
459
460
3.02M
    for (i = 0; i < 256; i++, s++)
461
3.01M
    {
462
3.01M
      s->Symbol = (Byte)i;
463
3.01M
      s->Freq = 1;
464
3.01M
      Ppmd8State_SetSuccessor(s, 0);
465
3.01M
    }
466
11.7k
  }
467
468
  
469
  
470
  
471
  
472
  
473
  
474
  
475
  
476
  
477
  
478
  
479
306k
  for (i = m = 0; m < 25; m++)
480
294k
  {
481
2.82M
    while (p->NS2Indx[i] == m)
482
2.53M
      i++;
483
2.64M
    for (k = 0; k < 8; k++)
484
2.35M
    {
485
2.35M
      unsigned r;
486
2.35M
      UInt16 *dest = p->BinSumm[m] + k;
487
2.35M
      const UInt16 val = (UInt16)(PPMD_BIN_SCALE - PPMD8_kInitBinEsc[k] / (i + 1));
488
21.1M
      for (r = 0; r < 64; r += 8)
489
18.8M
        dest[r] = val;
490
2.35M
    }
491
294k
  }
492
493
294k
  for (i = m = 0; m < 24; m++)
494
282k
  {
495
282k
    unsigned summ;
496
282k
    CPpmd_See *s;
497
3.28M
    while (p->NS2Indx[(size_t)i + 3] == m + 3)
498
3.00M
      i++;
499
282k
    s = p->See[m];
500
282k
    summ = ((2 * i + 5) << (PPMD_PERIOD_BITS - 4));
501
9.32M
    for (k = 0; k < 32; k++, s++)
502
9.04M
    {
503
9.04M
      s->Summ = (UInt16)summ;
504
9.04M
      s->Shift = (PPMD_PERIOD_BITS - 4);
505
9.04M
      s->Count = 7;
506
9.04M
    }
507
282k
  }
508
509
11.7k
  p->DummySee.Summ = 0; /* unused */
510
11.7k
  p->DummySee.Shift = PPMD_PERIOD_BITS;
511
11.7k
  p->DummySee.Count = 64; /* unused */
512
11.7k
}
513
514
515
void Ppmd8_Init(CPpmd8 *p, unsigned maxOrder, unsigned restoreMethod)
516
11.2k
{
517
11.2k
  p->MaxOrder = maxOrder;
518
11.2k
  p->RestoreMethod = restoreMethod;
519
11.2k
  Ppmd8_RestartModel(p);
520
11.2k
}
521
522
523
186k
#define FLAG_RESCALED  (1 << 2)
524
// #define FLAG_SYM_HIGH  (1 << 3)
525
10.5k
#define FLAG_PREV_HIGH (1 << 4)
526
527
86.3k
#define HiBits_Prepare(sym) ((unsigned)(sym) + 0xC0)
528
529
124M
#define HiBits_Convert_3(flags) (((flags) >> (8 - 3)) & (1 << 3))
530
32.0M
#define HiBits_Convert_4(flags) (((flags) >> (8 - 4)) & (1 << 4))
531
532
124M
#define PPMD8_HiBitsFlag_3(sym) HiBits_Convert_3(HiBits_Prepare(sym))
533
32.0M
#define PPMD8_HiBitsFlag_4(sym) HiBits_Convert_4(HiBits_Prepare(sym))
534
535
// #define PPMD8_HiBitsFlag_3(sym) (0x08 * ((sym) >= 0x40))
536
// #define PPMD8_HiBitsFlag_4(sym) (0x10 * ((sym) >= 0x40))
537
538
/*
539
Refresh() is called when we remove some symbols (successors) in context.
540
It increases Escape_Freq for sum of all removed symbols.
541
*/
542
543
static void Refresh(CPpmd8 *p, PPMD8_CTX_PTR ctx, unsigned oldNU, unsigned scale)
544
635
{
545
635
  unsigned i = ctx->NumStats, escFreq, sumFreq, flags;
546
635
  CPpmd_State *s = (CPpmd_State *)ShrinkUnits(p, STATS(ctx), oldNU, (i + 2) >> 1);
547
635
  ctx->Union4.Stats = REF(s);
548
549
  // #ifdef PPMD8_FREEZE_SUPPORT
550
  /*
551
    (ctx->Union2.SummFreq >= ((UInt32)1 << 15)) can be in FREEZE mode for some files.
552
    It's not good for range coder. So new versions of support fix:
553
       -   original PPMdI code rev.1
554
       +   original PPMdI code rev.2
555
       -   7-Zip default ((PPMD8_FREEZE_SUPPORT is not defined)
556
       +   7-Zip (p->RestoreMethod >= PPMD8_RESTORE_METHOD_FREEZE)
557
    if we       use that fixed line, we can lose compatibility with some files created before fix
558
    if we don't use that fixed line, the program can work incorrectly in FREEZE mode in rare case.
559
  */
560
  // if (p->RestoreMethod >= PPMD8_RESTORE_METHOD_FREEZE)
561
635
  {
562
635
    scale |= (ctx->Union2.SummFreq >= ((UInt32)1 << 15));
563
635
  }
564
  // #endif
565
566
567
568
635
  flags = HiBits_Prepare(s->Symbol);
569
635
  {
570
635
    unsigned freq = s->Freq;
571
635
    escFreq = ctx->Union2.SummFreq - freq;
572
635
    freq = (freq + scale) >> scale;
573
635
    sumFreq = freq;
574
635
    s->Freq = (Byte)freq;
575
635
  }
576
 
577
635
  do
578
85.7k
  {
579
85.7k
    unsigned freq = (++s)->Freq;
580
85.7k
    escFreq -= freq;
581
85.7k
    freq = (freq + scale) >> scale;
582
85.7k
    sumFreq += freq;
583
85.7k
    s->Freq = (Byte)freq;
584
85.7k
    flags |= HiBits_Prepare(s->Symbol);
585
85.7k
  }
586
85.7k
  while (--i);
587
  
588
635
  ctx->Union2.SummFreq = (UInt16)(sumFreq + ((escFreq + scale) >> scale));
589
635
  ctx->Flags = (Byte)((ctx->Flags & (FLAG_PREV_HIGH + FLAG_RESCALED * scale)) + HiBits_Convert_3(flags));
590
635
}
591
592
593
static void SWAP_STATES(CPpmd_State *t1, CPpmd_State *t2)
594
39.2M
{
595
39.2M
  CPpmd_State tmp = *t1;
596
39.2M
  *t1 = *t2;
597
39.2M
  *t2 = tmp;
598
39.2M
}
599
600
601
/*
602
CutOff() reduces contexts:
603
  It conversts Successors at MaxOrder to another Contexts to NULL-Successors
604
  It removes RAW-Successors and NULL-Successors that are not Order-0
605
      and it removes contexts when it has no Successors.
606
  if the (Union4.Stats) is close to (UnitsStart), it moves it up.
607
*/
608
609
static CPpmd_Void_Ref CutOff(CPpmd8 *p, PPMD8_CTX_PTR ctx, unsigned order)
610
0
{
611
0
  int ns = ctx->NumStats;
612
0
  unsigned nu;
613
0
  CPpmd_State *stats;
614
  
615
0
  if (ns == 0)
616
0
  {
617
0
    CPpmd_State *s = ONE_STATE(ctx);
618
0
    CPpmd_Void_Ref successor = SUCCESSOR(s);
619
0
    if ((Byte *)Ppmd8_GetPtr(p, successor) >= p->UnitsStart)
620
0
    {
621
0
      if (order < p->MaxOrder)
622
0
        successor = CutOff(p, CTX(successor), order + 1);
623
0
      else
624
0
        successor = 0;
625
0
      Ppmd8State_SetSuccessor(s, successor);
626
0
      if (successor || order <= 9) /* O_BOUND */
627
0
        return REF(ctx);
628
0
    }
629
0
    SpecialFreeUnit(p, ctx);
630
0
    return 0;
631
0
  }
632
633
0
  nu = ((unsigned)ns + 2) >> 1;
634
  // ctx->Union4.Stats = STATS_REF(MoveUnitsUp(p, STATS(ctx), nu));
635
0
  {
636
0
    unsigned indx = U2I(nu);
637
0
    stats = STATS(ctx);
638
639
0
    if ((UInt32)((Byte *)stats - p->UnitsStart) <= (1 << 14)
640
0
        && (CPpmd_Void_Ref)ctx->Union4.Stats <= p->FreeList[indx])
641
0
    {
642
0
      void *ptr = Ppmd8_RemoveNode(p, indx);
643
0
      ctx->Union4.Stats = STATS_REF(ptr);
644
0
      MEM_12_CPY(ptr, (const void *)stats, nu)
645
0
      if ((Byte *)stats != p->UnitsStart)
646
0
        Ppmd8_InsertNode(p, stats, indx);
647
0
      else
648
0
        p->UnitsStart += U2B(I2U(indx));
649
0
      stats = ptr;
650
0
    }
651
0
  }
652
653
0
  {
654
0
    CPpmd_State *s = stats + (unsigned)ns;
655
0
    do
656
0
    {
657
0
      CPpmd_Void_Ref successor = SUCCESSOR(s);
658
0
      if ((Byte *)Ppmd8_GetPtr(p, successor) < p->UnitsStart)
659
0
      {
660
0
        CPpmd_State *s2 = stats + (unsigned)(ns--);
661
0
        if (order)
662
0
        {
663
0
          if (s != s2)
664
0
            *s = *s2;
665
0
        }
666
0
        else
667
0
        {
668
0
          SWAP_STATES(s, s2);
669
0
          Ppmd8State_SetSuccessor(s2, 0);
670
0
        }
671
0
      }
672
0
      else
673
0
      {
674
0
        if (order < p->MaxOrder)
675
0
          Ppmd8State_SetSuccessor(s, CutOff(p, CTX(successor), order + 1));
676
0
        else
677
0
          Ppmd8State_SetSuccessor(s, 0);
678
0
      }
679
0
    }
680
0
    while (--s >= stats);
681
0
  }
682
  
683
0
  if (ns != ctx->NumStats && order)
684
0
  {
685
0
    if (ns < 0)
686
0
    {
687
0
      FreeUnits(p, stats, nu);
688
0
      SpecialFreeUnit(p, ctx);
689
0
      return 0;
690
0
    }
691
0
    ctx->NumStats = (Byte)ns;
692
0
    if (ns == 0)
693
0
    {
694
0
      const Byte sym = stats->Symbol;
695
0
      ctx->Flags = (Byte)((ctx->Flags & FLAG_PREV_HIGH) + PPMD8_HiBitsFlag_3(sym));
696
      // *ONE_STATE(ctx) = *stats;
697
0
      ctx->Union2.State2.Symbol = sym;
698
0
      ctx->Union2.State2.Freq = (Byte)(((unsigned)stats->Freq + 11) >> 3);
699
0
      ctx->Union4.State4.Successor_0 = stats->Successor_0;
700
0
      ctx->Union4.State4.Successor_1 = stats->Successor_1;
701
0
      FreeUnits(p, stats, nu);
702
0
    }
703
0
    else
704
0
    {
705
0
      Refresh(p, ctx, nu, ctx->Union2.SummFreq > 16 * (unsigned)ns);
706
0
    }
707
0
  }
708
  
709
0
  return REF(ctx);
710
0
}
711
712
713
714
#ifdef PPMD8_FREEZE_SUPPORT
715
716
/*
717
RemoveBinContexts()
718
  It conversts Successors at MaxOrder to another Contexts to NULL-Successors
719
  It changes RAW-Successors to NULL-Successors
720
  removes Bin Context without Successor, if suffix of that context is also binary.
721
*/
722
723
static CPpmd_Void_Ref RemoveBinContexts(CPpmd8 *p, PPMD8_CTX_PTR ctx, unsigned order)
724
{
725
  if (!ctx->NumStats)
726
  {
727
    CPpmd_State *s = ONE_STATE(ctx);
728
    CPpmd_Void_Ref successor = SUCCESSOR(s);
729
    if ((Byte *)Ppmd8_GetPtr(p, successor) >= p->UnitsStart && order < p->MaxOrder)
730
      successor = RemoveBinContexts(p, CTX(successor), order + 1);
731
    else
732
      successor = 0;
733
    Ppmd8State_SetSuccessor(s, successor);
734
    /* Suffix context can be removed already, since different (high-order)
735
       Successors may refer to same context. So we check Flags == 0xFF (Stamp == EMPTY_NODE) */
736
    if (!successor && (!SUFFIX(ctx)->NumStats || SUFFIX(ctx)->Flags == 0xFF))
737
    {
738
      FreeUnits(p, ctx, 1);
739
      return 0;
740
    }
741
  }
742
  else
743
  {
744
    CPpmd_State *s = STATS(ctx) + ctx->NumStats;
745
    do
746
    {
747
      CPpmd_Void_Ref successor = SUCCESSOR(s);
748
      if ((Byte *)Ppmd8_GetPtr(p, successor) >= p->UnitsStart && order < p->MaxOrder)
749
        Ppmd8State_SetSuccessor(s, RemoveBinContexts(p, CTX(successor), order + 1));
750
      else
751
        Ppmd8State_SetSuccessor(s, 0);
752
    }
753
    while (--s >= STATS(ctx));
754
  }
755
  
756
  return REF(ctx);
757
}
758
759
#endif
760
761
762
763
static UInt32 GetUsedMemory(const CPpmd8 *p)
764
0
{
765
0
  UInt32 v = 0;
766
0
  unsigned i;
767
0
  for (i = 0; i < PPMD_NUM_INDEXES; i++)
768
0
    v += p->Stamps[i] * I2U(i);
769
0
  return p->Size - (UInt32)(p->HiUnit - p->LoUnit) - (UInt32)(p->UnitsStart - p->Text) - U2B(v);
770
0
}
771
772
#ifdef PPMD8_FREEZE_SUPPORT
773
  #define RESTORE_MODEL(c1, fSuccessor) RestoreModel(p, c1, fSuccessor)
774
#else
775
513
  #define RESTORE_MODEL(c1, fSuccessor) RestoreModel(p, c1)
776
#endif
777
778
779
static void RestoreModel(CPpmd8 *p, PPMD8_CTX_PTR ctxError
780
    #ifdef PPMD8_FREEZE_SUPPORT
781
    , PPMD8_CTX_PTR fSuccessor
782
    #endif
783
    )
784
513
{
785
513
  PPMD8_CTX_PTR c;
786
513
  CPpmd_State *s;
787
513
  RESET_TEXT(0)
788
789
  // we go here in cases of error of allocation for context (c1)
790
  // Order(MinContext) < Order(ctxError) <= Order(MaxContext)
791
  
792
  // We remove last symbol from each of contexts [p->MaxContext ... ctxError) contexts
793
  // So we rollback all created (symbols) before error.
794
850
  for (c = p->MaxContext; c != ctxError; c = SUFFIX(c))
795
337
    if (--(c->NumStats) == 0)
796
150
    {
797
150
      s = STATS(c);
798
150
      c->Flags = (Byte)((c->Flags & FLAG_PREV_HIGH) + PPMD8_HiBitsFlag_3(s->Symbol));
799
      // *ONE_STATE(c) = *s;
800
150
      c->Union2.State2.Symbol = s->Symbol;
801
150
      c->Union2.State2.Freq = (Byte)(((unsigned)s->Freq + 11) >> 3);
802
150
      c->Union4.State4.Successor_0 = s->Successor_0;
803
150
      c->Union4.State4.Successor_1 = s->Successor_1;
804
805
150
      SpecialFreeUnit(p, s);
806
150
    }
807
187
    else
808
187
    {
809
      /* Refresh() can increase Escape_Freq on value of Freq of last symbol, that was added before error.
810
         so the largest possible increase for Escape_Freq is (8) from value before ModelUpoadet() */
811
187
      Refresh(p, c, ((unsigned)c->NumStats + 3) >> 1, 0);
812
187
    }
813
 
814
  // increase Escape Freq for context [ctxError ... p->MinContext)
815
1.01k
  for (; c != p->MinContext; c = SUFFIX(c))
816
504
    if (c->NumStats == 0)
817
21
    {
818
      // ONE_STATE(c)
819
21
      c->Union2.State2.Freq = (Byte)(((unsigned)c->Union2.State2.Freq + 1) >> 1);
820
21
    }
821
483
    else if ((c->Union2.SummFreq = (UInt16)(c->Union2.SummFreq + 4)) > 128 + 4 * c->NumStats)
822
448
      Refresh(p, c, ((unsigned)c->NumStats + 2) >> 1, 1);
823
824
  #ifdef PPMD8_FREEZE_SUPPORT
825
  if (p->RestoreMethod > PPMD8_RESTORE_METHOD_FREEZE)
826
  {
827
    p->MaxContext = fSuccessor;
828
    p->GlueCount += !(p->Stamps[1] & 1); // why?
829
  }
830
  else if (p->RestoreMethod == PPMD8_RESTORE_METHOD_FREEZE)
831
  {
832
    while (p->MaxContext->Suffix)
833
      p->MaxContext = SUFFIX(p->MaxContext);
834
    RemoveBinContexts(p, p->MaxContext, 0);
835
    // we change the current mode to (PPMD8_RESTORE_METHOD_FREEZE + 1)
836
    p->RestoreMethod = PPMD8_RESTORE_METHOD_FREEZE + 1;
837
    p->GlueCount = 0;
838
    p->OrderFall = p->MaxOrder;
839
  }
840
  else
841
  #endif
842
513
  if (p->RestoreMethod == PPMD8_RESTORE_METHOD_RESTART || GetUsedMemory(p) < (p->Size >> 1))
843
513
    Ppmd8_RestartModel(p);
844
0
  else
845
0
  {
846
0
    while (p->MaxContext->Suffix)
847
0
      p->MaxContext = SUFFIX(p->MaxContext);
848
0
    do
849
0
    {
850
0
      CutOff(p, p->MaxContext, 0);
851
0
      ExpandTextArea(p);
852
0
    }
853
0
    while (GetUsedMemory(p) > 3 * (p->Size >> 2));
854
0
    p->GlueCount = 0;
855
0
    p->OrderFall = p->MaxOrder;
856
0
  }
857
513
  p->MinContext = p->MaxContext;
858
513
}
859
860
861
862
Z7_NO_INLINE
863
static PPMD8_CTX_PTR Ppmd8_CreateSuccessors(CPpmd8 *p, BoolInt skip, CPpmd_State *s1, PPMD8_CTX_PTR c)
864
33.2M
{
865
866
33.2M
  CPpmd_Byte_Ref upBranch = (CPpmd_Byte_Ref)SUCCESSOR(p->FoundState);
867
33.2M
  Byte newSym, newFreq, flags;
868
33.2M
  unsigned numPs = 0;
869
33.2M
  CPpmd_State *ps[PPMD8_MAX_ORDER + 1]; /* fixed over Shkarin's code. Maybe it could work without + 1 too. */
870
  
871
33.2M
  if (!skip)
872
23.2M
    ps[numPs++] = p->FoundState;
873
  
874
62.5M
  while (c->Suffix)
875
61.9M
  {
876
61.9M
    CPpmd_Void_Ref successor;
877
61.9M
    CPpmd_State *s;
878
61.9M
    c = SUFFIX(c);
879
    
880
61.9M
    if (s1) { s = s1; s1 = NULL; }
881
29.2M
    else if (c->NumStats != 0)
882
15.9M
    {
883
15.9M
      Byte sym = p->FoundState->Symbol;
884
973M
      for (s = STATS(c); s->Symbol != sym; s++);
885
15.9M
      if (s->Freq < MAX_FREQ - 9) { s->Freq++; c->Union2.SummFreq++; }
886
15.9M
    }
887
13.3M
    else
888
13.3M
    {
889
13.3M
      s = ONE_STATE(c);
890
13.3M
      s->Freq = (Byte)(s->Freq + (!SUFFIX(c)->NumStats & (s->Freq < 24)));
891
13.3M
    }
892
61.9M
    successor = SUCCESSOR(s);
893
61.9M
    if (successor != upBranch)
894
32.5M
    {
895
896
32.5M
      c = CTX(successor);
897
32.5M
      if (numPs == 0)
898
1.15M
      {
899
        
900
       
901
1.15M
        return c;
902
1.15M
      }
903
31.4M
      break;
904
32.5M
    }
905
29.3M
    ps[numPs++] = s;
906
29.3M
  }
907
  
908
  
909
  
910
  
911
  
912
32.0M
  newSym = *(const Byte *)Ppmd8_GetPtr(p, upBranch);
913
32.0M
  upBranch++;
914
32.0M
  flags = (Byte)(PPMD8_HiBitsFlag_4(p->FoundState->Symbol) + PPMD8_HiBitsFlag_3(newSym));
915
  
916
32.0M
  if (c->NumStats == 0)
917
601k
    newFreq = c->Union2.State2.Freq;
918
31.4M
  else
919
31.4M
  {
920
31.4M
    UInt32 cf, s0;
921
31.4M
    CPpmd_State *s;
922
1.43G
    for (s = STATS(c); s->Symbol != newSym; s++);
923
31.4M
    cf = (UInt32)s->Freq - 1;
924
31.4M
    s0 = (UInt32)c->Union2.SummFreq - c->NumStats - cf;
925
    /*
926
    
927
928
      max(newFreq)= (s->Freq - 1), when (s0 == 1)
929
930
931
    */
932
31.4M
    newFreq = (Byte)(1 + ((2 * cf <= s0) ? (5 * cf > s0) : ((cf + 2 * s0 - 3) / s0)));
933
31.4M
  }
934
935
936
937
32.0M
  do
938
52.6M
  {
939
52.6M
    PPMD8_CTX_PTR c1;
940
    /* = AllocContext(p); */
941
52.6M
    if (p->HiUnit != p->LoUnit)
942
43.6M
      c1 = (PPMD8_CTX_PTR)(void *)(p->HiUnit -= UNIT_SIZE);
943
8.94M
    else if (p->FreeList[0] != 0)
944
3.79M
      c1 = (PPMD8_CTX_PTR)Ppmd8_RemoveNode(p, 0);
945
5.15M
    else
946
5.15M
    {
947
5.15M
      c1 = (PPMD8_CTX_PTR)Ppmd8_AllocUnitsRare(p, 0);
948
5.15M
      if (!c1)
949
7
        return NULL;
950
5.15M
    }
951
52.6M
    c1->Flags = flags;
952
52.6M
    c1->NumStats = 0;
953
52.6M
    c1->Union2.State2.Symbol = newSym;
954
52.6M
    c1->Union2.State2.Freq = newFreq;
955
52.6M
    Ppmd8State_SetSuccessor(ONE_STATE(c1), upBranch);
956
52.6M
    c1->Suffix = REF(c);
957
52.6M
    Ppmd8State_SetSuccessor(ps[--numPs], REF(c1));
958
52.6M
    c = c1;
959
52.6M
  }
960
52.6M
  while (numPs != 0);
961
  
962
32.0M
  return c;
963
32.0M
}
964
965
966
static PPMD8_CTX_PTR ReduceOrder(CPpmd8 *p, CPpmd_State *s1, PPMD8_CTX_PTR c)
967
817k
{
968
817k
  CPpmd_State *s = NULL;
969
817k
  PPMD8_CTX_PTR c1 = c;
970
817k
  CPpmd_Void_Ref upBranch = REF(p->Text);
971
  
972
  #ifdef PPMD8_FREEZE_SUPPORT
973
  /* The BUG in Shkarin's code was fixed: ps could overflow in CUT_OFF mode. */
974
  CPpmd_State *ps[PPMD8_MAX_ORDER + 1];
975
  unsigned numPs = 0;
976
  ps[numPs++] = p->FoundState;
977
  #endif
978
979
817k
  Ppmd8State_SetSuccessor(p->FoundState, upBranch);
980
817k
  p->OrderFall++;
981
982
817k
  for (;;)
983
817k
  {
984
817k
    if (s1)
985
0
    {
986
0
      c = SUFFIX(c);
987
0
      s = s1;
988
0
      s1 = NULL;
989
0
    }
990
817k
    else
991
817k
    {
992
817k
      if (!c->Suffix)
993
817k
      {
994
        #ifdef PPMD8_FREEZE_SUPPORT
995
        if (p->RestoreMethod > PPMD8_RESTORE_METHOD_FREEZE)
996
        {
997
          do { Ppmd8State_SetSuccessor(ps[--numPs], REF(c)); } while (numPs);
998
          RESET_TEXT(1)
999
          p->OrderFall = 1;
1000
        }
1001
        #endif
1002
817k
        return c;
1003
817k
      }
1004
0
      c = SUFFIX(c);
1005
0
      if (c->NumStats)
1006
0
      {
1007
0
        if ((s = STATS(c))->Symbol != p->FoundState->Symbol)
1008
0
          do { s++; } while (s->Symbol != p->FoundState->Symbol);
1009
0
        if (s->Freq < MAX_FREQ - 9)
1010
0
        {
1011
0
          s->Freq = (Byte)(s->Freq + 2);
1012
0
          c->Union2.SummFreq = (UInt16)(c->Union2.SummFreq + 2);
1013
0
        }
1014
0
      }
1015
0
      else
1016
0
      {
1017
0
        s = ONE_STATE(c);
1018
0
        s->Freq = (Byte)(s->Freq + (s->Freq < 32));
1019
0
      }
1020
0
    }
1021
0
    if (SUCCESSOR(s))
1022
0
      break;
1023
    #ifdef PPMD8_FREEZE_SUPPORT
1024
    ps[numPs++] = s;
1025
    #endif
1026
0
    Ppmd8State_SetSuccessor(s, upBranch);
1027
0
    p->OrderFall++;
1028
0
  }
1029
  
1030
  #ifdef PPMD8_FREEZE_SUPPORT
1031
  if (p->RestoreMethod > PPMD8_RESTORE_METHOD_FREEZE)
1032
  {
1033
    c = CTX(SUCCESSOR(s));
1034
    do { Ppmd8State_SetSuccessor(ps[--numPs], REF(c)); } while (numPs);
1035
    RESET_TEXT(1)
1036
    p->OrderFall = 1;
1037
    return c;
1038
  }
1039
  else
1040
  #endif
1041
0
  if (SUCCESSOR(s) <= upBranch)
1042
0
  {
1043
0
    PPMD8_CTX_PTR successor;
1044
0
    CPpmd_State *s2 = p->FoundState;
1045
0
    p->FoundState = s;
1046
1047
0
    successor = Ppmd8_CreateSuccessors(p, False, NULL, c);
1048
0
    if (!successor)
1049
0
      Ppmd8State_SetSuccessor(s, 0);
1050
0
    else
1051
0
      Ppmd8State_SetSuccessor(s, REF(successor));
1052
0
    p->FoundState = s2;
1053
0
  }
1054
  
1055
0
  {
1056
0
    CPpmd_Void_Ref successor = SUCCESSOR(s);
1057
0
    if (p->OrderFall == 1 && c1 == p->MaxContext)
1058
0
    {
1059
0
      Ppmd8State_SetSuccessor(p->FoundState, successor);
1060
0
      p->Text--;
1061
0
    }
1062
0
    if (successor == 0)
1063
0
      return NULL;
1064
0
    return CTX(successor);
1065
0
  }
1066
0
}
1067
1068
1069
1070
void Ppmd8_UpdateModel(CPpmd8 *p);
1071
Z7_NO_INLINE
1072
void Ppmd8_UpdateModel(CPpmd8 *p)
1073
102M
{
1074
102M
  CPpmd_Void_Ref maxSuccessor, minSuccessor = SUCCESSOR(p->FoundState);
1075
102M
  PPMD8_CTX_PTR c;
1076
102M
  unsigned s0, ns, fFreq = p->FoundState->Freq;
1077
102M
  Byte flag, fSymbol = p->FoundState->Symbol;
1078
102M
  {
1079
102M
  CPpmd_State *s = NULL;
1080
102M
  if (p->FoundState->Freq < MAX_FREQ / 4 && p->MinContext->Suffix != 0)
1081
55.5M
  {
1082
    /* Update Freqs in Suffix Context */
1083
1084
55.5M
    c = SUFFIX(p->MinContext);
1085
    
1086
55.5M
    if (c->NumStats == 0)
1087
7.31M
    {
1088
7.31M
      s = ONE_STATE(c);
1089
7.31M
      if (s->Freq < 32)
1090
7.31M
        s->Freq++;
1091
7.31M
    }
1092
48.2M
    else
1093
48.2M
    {
1094
48.2M
      Byte sym = p->FoundState->Symbol;
1095
48.2M
      s = STATS(c);
1096
1097
48.2M
      if (s->Symbol != sym)
1098
45.7M
      {
1099
45.7M
        do
1100
5.15G
        {
1101
        
1102
5.15G
          s++;
1103
5.15G
        }
1104
5.15G
        while (s->Symbol != sym);
1105
        
1106
45.7M
        if (s[0].Freq >= s[-1].Freq)
1107
23.6M
        {
1108
23.6M
          SWAP_STATES(&s[0], &s[-1]);
1109
23.6M
          s--;
1110
23.6M
        }
1111
45.7M
      }
1112
      
1113
48.2M
      if (s->Freq < MAX_FREQ - 9)
1114
45.2M
      {
1115
45.2M
        s->Freq = (Byte)(s->Freq + 2);
1116
45.2M
        c->Union2.SummFreq = (UInt16)(c->Union2.SummFreq + 2);
1117
45.2M
      }
1118
48.2M
    }
1119
55.5M
  }
1120
  
1121
102M
  c = p->MaxContext;
1122
102M
  if (p->OrderFall == 0 && minSuccessor)
1123
9.93M
  {
1124
9.93M
    PPMD8_CTX_PTR cs = Ppmd8_CreateSuccessors(p, True, s, p->MinContext);
1125
9.93M
    if (!cs)
1126
5
    {
1127
5
      Ppmd8State_SetSuccessor(p->FoundState, 0);
1128
5
      RESTORE_MODEL(c, CTX(minSuccessor));
1129
5
      return;
1130
5
    }
1131
9.93M
    Ppmd8State_SetSuccessor(p->FoundState, REF(cs));
1132
9.93M
    p->MinContext = p->MaxContext = cs;
1133
9.93M
    return;
1134
9.93M
  }
1135
  
1136
1137
1138
1139
92.4M
  {
1140
92.4M
    Byte *text = p->Text;
1141
92.4M
    *text++ = p->FoundState->Symbol;
1142
92.4M
    p->Text = text;
1143
92.4M
    if (text >= p->UnitsStart)
1144
60
    {
1145
60
      RESTORE_MODEL(c, CTX(minSuccessor)); /* check it */
1146
60
      return;
1147
60
    }
1148
92.4M
    maxSuccessor = REF(text);
1149
92.4M
  }
1150
1151
92.4M
  if (!minSuccessor)
1152
817k
  {
1153
817k
    PPMD8_CTX_PTR cs = ReduceOrder(p, s, p->MinContext);
1154
817k
    if (!cs)
1155
0
    {
1156
0
      RESTORE_MODEL(c, NULL);
1157
0
      return;
1158
0
    }
1159
817k
    minSuccessor = REF(cs);
1160
817k
  }
1161
91.6M
  else if ((Byte *)Ppmd8_GetPtr(p, minSuccessor) < p->UnitsStart)
1162
23.2M
  {
1163
23.2M
    PPMD8_CTX_PTR cs = Ppmd8_CreateSuccessors(p, False, s, p->MinContext);
1164
23.2M
    if (!cs)
1165
2
    {
1166
2
      RESTORE_MODEL(c, NULL);
1167
2
      return;
1168
2
    }
1169
23.2M
    minSuccessor = REF(cs);
1170
23.2M
  }
1171
  
1172
92.4M
  if (--p->OrderFall == 0)
1173
27.7M
  {
1174
27.7M
    maxSuccessor = minSuccessor;
1175
27.7M
    p->Text -= (p->MaxContext != p->MinContext);
1176
27.7M
  }
1177
  #ifdef PPMD8_FREEZE_SUPPORT
1178
  else if (p->RestoreMethod > PPMD8_RESTORE_METHOD_FREEZE)
1179
  {
1180
    maxSuccessor = minSuccessor;
1181
    RESET_TEXT(0)
1182
    p->OrderFall = 0;
1183
  }
1184
  #endif
1185
92.4M
  }
1186
  
1187
  
1188
  
1189
  
1190
  
1191
  
1192
  
1193
  
1194
  
1195
  
1196
  
1197
  
1198
  
1199
  
1200
  
1201
  
1202
  
1203
  
1204
  
1205
  
1206
  
1207
  
1208
  
1209
  
1210
  
1211
1212
  
1213
  
1214
92.4M
  flag = (Byte)(PPMD8_HiBitsFlag_3(fSymbol));
1215
92.4M
  s0 = p->MinContext->Union2.SummFreq - (ns = p->MinContext->NumStats) - fFreq;
1216
  
1217
184M
  for (; c != p->MinContext; c = SUFFIX(c))
1218
91.6M
  {
1219
91.6M
    unsigned ns1;
1220
91.6M
    UInt32 sum;
1221
    
1222
91.6M
    if ((ns1 = c->NumStats) != 0)
1223
67.3M
    {
1224
67.3M
      if ((ns1 & 1) != 0)
1225
37.3M
      {
1226
        /* Expand for one UNIT */
1227
37.3M
        const unsigned oldNU = (ns1 + 1) >> 1;
1228
37.3M
        const unsigned i = U2I(oldNU);
1229
37.3M
        if (i != U2I((size_t)oldNU + 1))
1230
22.6M
        {
1231
22.6M
          void *ptr = Ppmd8_AllocUnits(p, i + 1);
1232
22.6M
          void *oldPtr;
1233
22.6M
          if (!ptr)
1234
444
          {
1235
444
            RESTORE_MODEL(c, CTX(minSuccessor));
1236
444
            return;
1237
444
          }
1238
22.6M
          oldPtr = STATS(c);
1239
22.6M
          MEM_12_CPY(ptr, oldPtr, oldNU)
1240
22.6M
          Ppmd8_InsertNode(p, oldPtr, i);
1241
22.6M
          c->Union4.Stats = STATS_REF(ptr);
1242
22.6M
        }
1243
37.3M
      }
1244
67.3M
      sum = c->Union2.SummFreq;
1245
      /* max increase of Escape_Freq is 1 here.
1246
         an average increase is 1/3 per symbol */
1247
67.3M
      sum += (UInt32)(unsigned)(3 * ns1 + 1 < ns);
1248
      /* original PPMdH uses 16-bit variable for (sum) here.
1249
         But (sum < ???). Do we need to truncate (sum) to 16-bit */
1250
      // sum = (UInt16)sum;
1251
67.3M
    }
1252
24.3M
    else
1253
24.3M
    {
1254
      
1255
24.3M
      CPpmd_State *s = (CPpmd_State*)Ppmd8_AllocUnits(p, 0);
1256
24.3M
      if (!s)
1257
2
      {
1258
2
        RESTORE_MODEL(c, CTX(minSuccessor));
1259
2
        return;
1260
2
      }
1261
24.3M
      {
1262
24.3M
        unsigned freq = c->Union2.State2.Freq;
1263
        // s = *ONE_STATE(c);
1264
24.3M
        s->Symbol = c->Union2.State2.Symbol;
1265
24.3M
        s->Successor_0 = c->Union4.State4.Successor_0;
1266
24.3M
        s->Successor_1 = c->Union4.State4.Successor_1;
1267
        // Ppmd8State_SetSuccessor(s, c->Union4.Stats);  // call it only for debug purposes to check the order of
1268
                                              // (Successor_0 and Successor_1) in LE/BE.
1269
24.3M
        c->Union4.Stats = REF(s);
1270
24.3M
        if (freq < MAX_FREQ / 4 - 1)
1271
24.3M
          freq <<= 1;
1272
13.6k
        else
1273
13.6k
          freq = MAX_FREQ - 4;
1274
1275
24.3M
        s->Freq = (Byte)freq;
1276
1277
24.3M
        sum = (UInt32)(freq + p->InitEsc + (ns > 2));   // Ppmd8 (> 2)
1278
24.3M
      }
1279
24.3M
    }
1280
1281
91.6M
    {
1282
91.6M
      CPpmd_State *s = STATS(c) + ns1 + 1;
1283
91.6M
      UInt32 cf = 2 * (sum + 6) * (UInt32)fFreq;
1284
91.6M
      UInt32 sf = (UInt32)s0 + sum;
1285
91.6M
      s->Symbol = fSymbol;
1286
91.6M
      c->NumStats = (Byte)(ns1 + 1);
1287
91.6M
      Ppmd8State_SetSuccessor(s, maxSuccessor);
1288
91.6M
      c->Flags |= flag;
1289
91.6M
      if (cf < 6 * sf)
1290
75.2M
      {
1291
75.2M
        cf = (unsigned)1 + (cf > sf) + (cf >= 4 * sf);
1292
75.2M
        sum += 4;
1293
        /* It can add (1, 2, 3) to Escape_Freq */
1294
75.2M
      }
1295
16.4M
      else
1296
16.4M
      {
1297
16.4M
        cf = (unsigned)4 + (cf > 9 * sf) + (cf > 12 * sf) + (cf > 15 * sf);
1298
16.4M
        sum += cf;
1299
16.4M
      }
1300
1301
91.6M
      c->Union2.SummFreq = (UInt16)sum;
1302
91.6M
      s->Freq = (Byte)cf;
1303
91.6M
    }
1304
1305
91.6M
  }
1306
92.4M
  p->MaxContext = p->MinContext = CTX(minSuccessor);
1307
92.4M
}
1308
  
1309
1310
1311
Z7_NO_INLINE
1312
static void Ppmd8_Rescale(CPpmd8 *p)
1313
195k
{
1314
195k
  unsigned i, adder, sumFreq, escFreq;
1315
195k
  CPpmd_State *stats = STATS(p->MinContext);
1316
195k
  CPpmd_State *s = p->FoundState;
1317
1318
  /* Sort the list by Freq */
1319
195k
  if (s != stats)
1320
25.6k
  {
1321
25.6k
    CPpmd_State tmp = *s;
1322
25.6k
    do
1323
1.71M
      s[0] = s[-1];
1324
1.71M
    while (--s != stats);
1325
25.6k
    *s = tmp;
1326
25.6k
  }
1327
1328
195k
  sumFreq = s->Freq;
1329
195k
  escFreq = p->MinContext->Union2.SummFreq - sumFreq;
1330
1331
1332
  
1333
  
1334
  
1335
  
1336
195k
  adder = (p->OrderFall != 0);
1337
  
1338
  #ifdef PPMD8_FREEZE_SUPPORT
1339
  adder |= (p->RestoreMethod > PPMD8_RESTORE_METHOD_FREEZE);
1340
  #endif
1341
  
1342
195k
  sumFreq = (sumFreq + 4 + adder) >> 1;
1343
195k
  i = p->MinContext->NumStats;
1344
195k
  s->Freq = (Byte)sumFreq;
1345
  
1346
195k
  do
1347
23.5M
  {
1348
23.5M
    unsigned freq = (++s)->Freq;
1349
23.5M
    escFreq -= freq;
1350
23.5M
    freq = (freq + adder) >> 1;
1351
23.5M
    sumFreq += freq;
1352
23.5M
    s->Freq = (Byte)freq;
1353
23.5M
    if (freq > s[-1].Freq)
1354
9.89M
    {
1355
9.89M
      CPpmd_State tmp = *s;
1356
9.89M
      CPpmd_State *s1 = s;
1357
9.89M
      do
1358
551M
      {
1359
551M
        s1[0] = s1[-1];
1360
551M
      }
1361
551M
      while (--s1 != stats && freq > s1[-1].Freq);
1362
9.89M
      *s1 = tmp;
1363
9.89M
    }
1364
23.5M
  }
1365
23.5M
  while (--i);
1366
  
1367
195k
  if (s->Freq == 0)
1368
49.0k
  {
1369
    /* Remove all items with Freq == 0 */
1370
49.0k
    CPpmd8_Context *mc;
1371
49.0k
    unsigned numStats, numStatsNew, n0, n1;
1372
    
1373
323k
    i = 0; do { i++; } while ((--s)->Freq == 0);
1374
    
1375
    
1376
1377
    
1378
49.0k
    escFreq += i;
1379
49.0k
    mc = p->MinContext;
1380
49.0k
    numStats = mc->NumStats;
1381
49.0k
    numStatsNew = numStats - i;
1382
49.0k
    mc->NumStats = (Byte)(numStatsNew);
1383
49.0k
    n0 = (numStats + 2) >> 1;
1384
    
1385
49.0k
    if (numStatsNew == 0)
1386
9.77k
    {
1387
    
1388
9.77k
      unsigned freq = (2 * (unsigned)stats->Freq + escFreq - 1) / escFreq;
1389
9.77k
      if (freq > MAX_FREQ / 3)
1390
3.49k
        freq = MAX_FREQ / 3;
1391
9.77k
      mc->Flags = (Byte)((mc->Flags & FLAG_PREV_HIGH) + PPMD8_HiBitsFlag_3(stats->Symbol));
1392
      
1393
      
1394
      
1395
      
1396
      
1397
9.77k
      s = ONE_STATE(mc);
1398
9.77k
      *s = *stats;
1399
9.77k
      s->Freq = (Byte)freq;
1400
9.77k
      p->FoundState = s;
1401
9.77k
      Ppmd8_InsertNode(p, stats, U2I(n0));
1402
9.77k
      return;
1403
9.77k
    }
1404
1405
39.2k
    n1 = (numStatsNew + 2) >> 1;
1406
39.2k
    if (n0 != n1)
1407
34.4k
      mc->Union4.Stats = STATS_REF(ShrinkUnits(p, stats, n0, n1));
1408
39.2k
    {
1409
      // here we are for max order only. So Ppmd8_MakeEscFreq() doesn't use mc->Flags
1410
      // but we still need current (Flags & FLAG_PREV_HIGH), if we will convert context to 1-symbol context later.
1411
      /*
1412
      unsigned flags = HiBits_Prepare((s = STATS(mc))->Symbol);
1413
      i = mc->NumStats;
1414
      do { flags |= HiBits_Prepare((++s)->Symbol); } while (--i);
1415
      mc->Flags = (Byte)((mc->Flags & ~FLAG_SYM_HIGH) + HiBits_Convert_3(flags));
1416
      */
1417
39.2k
    }
1418
39.2k
  }
1419
1420
1421
1422
  
1423
1424
1425
185k
  {
1426
185k
    CPpmd8_Context *mc = p->MinContext;
1427
185k
    mc->Union2.SummFreq = (UInt16)(sumFreq + escFreq - (escFreq >> 1));
1428
185k
    mc->Flags |= FLAG_RESCALED;
1429
185k
    p->FoundState = STATS(mc);
1430
185k
  }
1431
185k
}
1432
1433
1434
CPpmd_See *Ppmd8_MakeEscFreq(CPpmd8 *p, unsigned numMasked1, UInt32 *escFreq)
1435
89.7M
{
1436
89.7M
  CPpmd_See *see;
1437
89.7M
  const CPpmd8_Context *mc = p->MinContext;
1438
89.7M
  unsigned numStats = mc->NumStats;
1439
89.7M
  if (numStats != 0xFF)
1440
45.1M
  {
1441
    // (3 <= numStats + 2 <= 256)   (3 <= NS2Indx[3] and NS2Indx[256] === 26)
1442
45.1M
    see = p->See[(size_t)(unsigned)p->NS2Indx[(size_t)numStats + 2] - 3]
1443
45.1M
        + (mc->Union2.SummFreq > 11 * (numStats + 1))
1444
45.1M
        + 2 * (unsigned)(2 * numStats < ((unsigned)SUFFIX(mc)->NumStats + numMasked1))
1445
45.1M
        + mc->Flags;
1446
1447
45.1M
    {
1448
      // if (see->Summ) field is larger than 16-bit, we need only low 16 bits of Summ
1449
45.1M
      const unsigned summ = (UInt16)see->Summ; // & 0xFFFF
1450
45.1M
      const unsigned r = (summ >> see->Shift);
1451
45.1M
      see->Summ = (UInt16)(summ - r);
1452
45.1M
      *escFreq = (UInt32)(r + (r == 0));
1453
45.1M
    }
1454
45.1M
  }
1455
44.5M
  else
1456
44.5M
  {
1457
44.5M
    see = &p->DummySee;
1458
44.5M
    *escFreq = 1;
1459
44.5M
  }
1460
89.7M
  return see;
1461
89.7M
}
1462
1463
 
1464
static void Ppmd8_NextContext(CPpmd8 *p)
1465
24.1M
{
1466
24.1M
  PPMD8_CTX_PTR c = CTX(SUCCESSOR(p->FoundState));
1467
24.1M
  if (p->OrderFall == 0 && (const Byte *)c >= p->UnitsStart)
1468
5.07M
    p->MaxContext = p->MinContext = c;
1469
19.0M
  else
1470
19.0M
    Ppmd8_UpdateModel(p);
1471
24.1M
}
1472
 
1473
1474
void Ppmd8_Update1(CPpmd8 *p)
1475
20.0M
{
1476
20.0M
  CPpmd_State *s = p->FoundState;
1477
20.0M
  unsigned freq = s->Freq;
1478
20.0M
  freq += 4;
1479
20.0M
  p->MinContext->Union2.SummFreq = (UInt16)(p->MinContext->Union2.SummFreq + 4);
1480
20.0M
  s->Freq = (Byte)freq;
1481
20.0M
  if (freq > s[-1].Freq)
1482
15.6M
  {
1483
15.6M
    SWAP_STATES(s, &s[-1]);
1484
15.6M
    p->FoundState = --s;
1485
15.6M
    if (freq > MAX_FREQ)
1486
2.51k
      Ppmd8_Rescale(p);
1487
15.6M
  }
1488
20.0M
  Ppmd8_NextContext(p);
1489
20.0M
}
1490
1491
1492
void Ppmd8_Update1_0(CPpmd8 *p)
1493
4.15M
{
1494
4.15M
  CPpmd_State *s = p->FoundState;
1495
4.15M
  CPpmd8_Context *mc = p->MinContext;
1496
4.15M
  unsigned freq = s->Freq;
1497
4.15M
  const unsigned summFreq = mc->Union2.SummFreq;
1498
4.15M
  p->PrevSuccess = (2 * freq >= summFreq); // Ppmd8 (>=)
1499
4.15M
  p->RunLength += (Int32)p->PrevSuccess;
1500
4.15M
  mc->Union2.SummFreq = (UInt16)(summFreq + 4);
1501
4.15M
  freq += 4;
1502
4.15M
  s->Freq = (Byte)freq;
1503
4.15M
  if (freq > MAX_FREQ)
1504
107k
    Ppmd8_Rescale(p);
1505
4.15M
  Ppmd8_NextContext(p);
1506
4.15M
}
1507
1508
1509
/*
1510
void Ppmd8_UpdateBin(CPpmd8 *p)
1511
{
1512
  unsigned freq = p->FoundState->Freq;
1513
  p->FoundState->Freq = (Byte)(freq + (freq < 196)); // Ppmd8 (196)
1514
  p->PrevSuccess = 1;
1515
  p->RunLength++;
1516
  Ppmd8_NextContext(p);
1517
}
1518
*/
1519
1520
void Ppmd8_Update2(CPpmd8 *p)
1521
72.7M
{
1522
72.7M
  CPpmd_State *s = p->FoundState;
1523
72.7M
  unsigned freq = s->Freq;
1524
72.7M
  freq += 4;
1525
72.7M
  p->RunLength = p->InitRL;
1526
72.7M
  p->MinContext->Union2.SummFreq = (UInt16)(p->MinContext->Union2.SummFreq + 4);
1527
72.7M
  s->Freq = (Byte)freq;
1528
72.7M
  if (freq > MAX_FREQ)
1529
85.2k
    Ppmd8_Rescale(p);
1530
72.7M
  Ppmd8_UpdateModel(p);
1531
72.7M
}
1532
1533
/* H->I changes:
1534
  NS2Indx
1535
  GlueCount, and Glue method
1536
  BinSum
1537
  See / EscFreq
1538
  Ppmd8_CreateSuccessors updates more suffix contexts
1539
  Ppmd8_UpdateModel consts.
1540
  PrevSuccess Update
1541
1542
Flags:
1543
  (1 << 2) - the Context was Rescaled
1544
  (1 << 3) - there is symbol in Stats with (sym >= 0x40) in
1545
  (1 << 4) - main symbol of context is (sym >= 0x40)
1546
*/
1547
1548
#undef RESET_TEXT
1549
#undef FLAG_RESCALED
1550
#undef FLAG_PREV_HIGH
1551
#undef HiBits_Prepare
1552
#undef HiBits_Convert_3
1553
#undef HiBits_Convert_4
1554
#undef PPMD8_HiBitsFlag_3
1555
#undef PPMD8_HiBitsFlag_4
1556
#undef RESTORE_MODEL
1557
1558
#undef MAX_FREQ
1559
#undef UNIT_SIZE
1560
#undef U2B
1561
#undef U2I
1562
#undef I2U
1563
1564
#undef REF
1565
#undef STATS_REF
1566
#undef CTX
1567
#undef STATS
1568
#undef ONE_STATE
1569
#undef SUFFIX
1570
#undef NODE
1571
#undef EMPTY_NODE
1572
#undef MEM_12_CPY
1573
#undef SUCCESSOR
1574
#undef SWAP_STATES