/src/c-blosc2/blosc/stune.c

Source (jump to first uncovered line)
/*********************************************************************
  Blosc - Blocked Shuffling and Compression Library

  Copyright (c) 2021  Blosc Development Team <blosc@blosc.org>
  https://blosc.org
  License: BSD 3-Clause (see LICENSE.txt)

  See LICENSE.txt for details about copyright and rights to use.
**********************************************************************/

#include "stune.h"

#include <stdbool.h>
#include <stdio.h>


/* Whether a codec is meant for High Compression Ratios
   Includes LZ4 + BITSHUFFLE here, but not BloscLZ + BITSHUFFLE because,
   for some reason, the latter does not work too well */
static bool is_HCR(blosc2_context * context) {
  switch (context->compcode) {
    case BLOSC_BLOSCLZ :
      return false;
    case BLOSC_LZ4 :
      // return (context->filter_flags & BLOSC_DOBITSHUFFLE) ? true : false;
      // Do not treat LZ4 differently than BloscLZ here
      return false;
    case BLOSC_LZ4HC :
    case BLOSC_ZLIB :
    case BLOSC_ZSTD :
      return true;
    default :
      return false;
  }
}

int blosc_stune_init(void * config, blosc2_context* cctx, blosc2_context* dctx) {
  BLOSC_UNUSED_PARAM(config);
  BLOSC_UNUSED_PARAM(cctx);
  BLOSC_UNUSED_PARAM(dctx);

  return BLOSC2_ERROR_SUCCESS;
}

// Set the automatic blocksize 0 to its real value
int blosc_stune_next_blocksize(blosc2_context *context) {
  int32_t clevel = context->clevel;
  int32_t typesize = context->typesize;
  int32_t nbytes = context->sourcesize;
  int32_t user_blocksize = context->blocksize;
  int32_t blocksize = nbytes;

  // Protection against very small buffers
  if (nbytes < typesize) {
    context->blocksize = 1;
    return BLOSC2_ERROR_SUCCESS;
  }

  int splitmode = split_block(context, typesize, blocksize);
  if (user_blocksize) {
    blocksize = user_blocksize;
    goto last;
  }

  if (nbytes >= L1) {
    blocksize = L1;

    /* For HCR codecs, increase the block sizes by a factor of 2 because they
        are meant for compressing large blocks (i.e. they show a big overhead
        when compressing small ones). */
    if (is_HCR(context)) {
      blocksize *= 2;
    }

    // Choose a different blocksize depending on the compression level
    switch (clevel) {
      case 0:
        // Case of plain copy
        blocksize /= 4;
        break;
      case 1:
        blocksize /= 2;
        break;
      case 2:
        blocksize *= 1;
        break;
      case 3:
        blocksize *= 2;
        break;
      case 4:
      case 5:
        blocksize *= 4;
        break;
      case 6:
      case 7:
      case 8:
        blocksize *= 8;
        break;
      case 9:
        // Do not exceed 256 KB for non HCR codecs
        blocksize *= 8;
        if (is_HCR(context)) {
          blocksize *= 2;
        }
        break;
      default:
        break;
    }
  }

  /* Now the blocksize for splittable codecs */
  if (clevel > 0 && splitmode) {
    // For performance reasons, do not exceed 256 KB (it must fit in L2 cache)
    switch (clevel) {
      case 1:
      case 2:
      case 3:
        blocksize = 32 * 1024;
        break;
      case 4:
      case 5:
      case 6:
        blocksize = 64 * 1024;
        break;
      case 7:
        blocksize = 128 * 1024;
        break;
      case 8:
        blocksize = 256 * 1024;
        break;
      case 9:
      default:
        blocksize = 512 * 1024;
        break;
    }
    // Multiply by typesize to get proper split sizes
    blocksize *= typesize;
    // But do not exceed 4 MB per thread (having this capacity in L3 is normal in modern CPUs)
    if (blocksize > 4 * 1024 * 1024) {
      blocksize = 4 * 1024 * 1024;
    }
    if (blocksize < 32 * 1024) {
      /* Do not use a too small blocksize (< 32 KB) when typesize is small */
      blocksize = 32 * 1024;
    }
  }

  last:
  /* Check that blocksize is not too large */
  if (blocksize > nbytes) {
    blocksize = nbytes;
  }

  // blocksize *must absolutely* be a multiple of the typesize
  if (blocksize > typesize) {
    blocksize = blocksize / typesize * typesize;
  }

  context->blocksize = blocksize;
  BLOSC_INFO("compcode: %d, clevel: %d, blocksize: %d, splitmode: %d, typesize: %d",
             context->compcode, context->clevel, blocksize, splitmode, typesize);

  return BLOSC2_ERROR_SUCCESS;
}

int blosc_stune_next_cparams(blosc2_context * context) {
  BLOSC_UNUSED_PARAM(context);

  return BLOSC2_ERROR_SUCCESS;
}

int blosc_stune_update(blosc2_context * context, double ctime) {
  BLOSC_UNUSED_PARAM(context);
  BLOSC_UNUSED_PARAM(ctime);

  return BLOSC2_ERROR_SUCCESS;
}

int blosc_stune_free(blosc2_context * context) {
  BLOSC_UNUSED_PARAM(context);

  return BLOSC2_ERROR_SUCCESS;
}

int split_block(blosc2_context *context, int32_t typesize, int32_t blocksize) {
  switch (context->splitmode) {
    case BLOSC_ALWAYS_SPLIT:
      return 1;
    case BLOSC_NEVER_SPLIT:
      return 0;
    case BLOSC_FORWARD_COMPAT_SPLIT:
    case BLOSC_AUTO_SPLIT:
      // These cases will be handled later
      break;
    default:
      BLOSC_TRACE_WARNING("Unrecognized split mode.  Default to BLOSC_FORWARD_COMPAT_SPLIT");
  }

  int compcode = context->compcode;
  return (
          // Fast codecs like blosclz, lz4 seems to prefer to split
          ((compcode == BLOSC_BLOSCLZ) || (compcode == BLOSC_LZ4)
            // and low levels of zstd too
            || ((compcode == BLOSC_ZSTD) && (context->clevel <= 5))
            ) &&
          // ...but split seems to harm cratio too much when not using shuffle
          (context->filter_flags & BLOSC_DOSHUFFLE) &&
          (typesize <= MAX_STREAMS) &&
          (blocksize / typesize) >= BLOSC_MIN_BUFFERSIZE);
}

Coverage Report

Created: 2025-06-20 06:08

Line	Count	Source (jump to first uncovered line)
1		/*********************************************************************
2		Blosc - Blocked Shuffling and Compression Library
3
4		Copyright (c) 2021 Blosc Development Team <blosc@blosc.org>
5		https://blosc.org
6		License: BSD 3-Clause (see LICENSE.txt)
7
8		See LICENSE.txt for details about copyright and rights to use.
9		**********************************************************************/
10
11		#include "stune.h"
12
13		#include <stdbool.h>
14		#include <stdio.h>
15
16
17		/* Whether a codec is meant for High Compression Ratios
18		Includes LZ4 + BITSHUFFLE here, but not BloscLZ + BITSHUFFLE because,
19		for some reason, the latter does not work too well */
20	0	static bool is_HCR(blosc2_context * context) {
21	0	switch (context->compcode) {
22	0	case BLOSC_BLOSCLZ :
23	0	return false;
24	0	case BLOSC_LZ4 :
25		// return (context->filter_flags & BLOSC_DOBITSHUFFLE) ? true : false;
26		// Do not treat LZ4 differently than BloscLZ here
27	0	return false;
28	0	case BLOSC_LZ4HC :
29	0	case BLOSC_ZLIB :
30	0	case BLOSC_ZSTD :
31	0	return true;
32	0	default :
33	0	return false;
34	0	}
35	0	}
36
37	0	int blosc_stune_init(void * config, blosc2_context* cctx, blosc2_context* dctx) {
38	0	BLOSC_UNUSED_PARAM(config);
39	0	BLOSC_UNUSED_PARAM(cctx);
40	0	BLOSC_UNUSED_PARAM(dctx);
41
42	0	return BLOSC2_ERROR_SUCCESS;
43	0	}
44
45		// Set the automatic blocksize 0 to its real value
46	0	int blosc_stune_next_blocksize(blosc2_context *context) {
47	0	int32_t clevel = context->clevel;
48	0	int32_t typesize = context->typesize;
49	0	int32_t nbytes = context->sourcesize;
50	0	int32_t user_blocksize = context->blocksize;
51	0	int32_t blocksize = nbytes;
52
53		// Protection against very small buffers
54	0	if (nbytes < typesize) {
55	0	context->blocksize = 1;
56	0	return BLOSC2_ERROR_SUCCESS;
57	0	}
58
59	0	int splitmode = split_block(context, typesize, blocksize);
60	0	if (user_blocksize) {
61	0	blocksize = user_blocksize;
62	0	goto last;
63	0	}
64
65	0	if (nbytes >= L1) {
66	0	blocksize = L1;
67
68		/* For HCR codecs, increase the block sizes by a factor of 2 because they
69		are meant for compressing large blocks (i.e. they show a big overhead
70		when compressing small ones). */
71	0	if (is_HCR(context)) {
72	0	blocksize *= 2;
73	0	}
74
75		// Choose a different blocksize depending on the compression level
76	0	switch (clevel) {
77	0	case 0:
78		// Case of plain copy
79	0	blocksize /= 4;
80	0	break;
81	0	case 1:
82	0	blocksize /= 2;
83	0	break;
84	0	case 2:
85	0	blocksize *= 1;
86	0	break;
87	0	case 3:
88	0	blocksize *= 2;
89	0	break;
90	0	case 4:
91	0	case 5:
92	0	blocksize *= 4;
93	0	break;
94	0	case 6:
95	0	case 7:
96	0	case 8:
97	0	blocksize *= 8;
98	0	break;
99	0	case 9:
100		// Do not exceed 256 KB for non HCR codecs
101	0	blocksize *= 8;
102	0	if (is_HCR(context)) {
103	0	blocksize *= 2;
104	0	}
105	0	break;
106	0	default:
107	0	break;
108	0	}
109	0	}
110
111		/* Now the blocksize for splittable codecs */
112	0	if (clevel > 0 && splitmode) {
113		// For performance reasons, do not exceed 256 KB (it must fit in L2 cache)
114	0	switch (clevel) {
115	0	case 1:
116	0	case 2:
117	0	case 3:
118	0	blocksize = 32 * 1024;
119	0	break;
120	0	case 4:
121	0	case 5:
122	0	case 6:
123	0	blocksize = 64 * 1024;
124	0	break;
125	0	case 7:
126	0	blocksize = 128 * 1024;
127	0	break;
128	0	case 8:
129	0	blocksize = 256 * 1024;
130	0	break;
131	0	case 9:
132	0	default:
133	0	blocksize = 512 * 1024;
134	0	break;
135	0	}
136		// Multiply by typesize to get proper split sizes
137	0	blocksize *= typesize;
138		// But do not exceed 4 MB per thread (having this capacity in L3 is normal in modern CPUs)
139	0	if (blocksize > 4 * 1024 * 1024) {
140	0	blocksize = 4 * 1024 * 1024;
141	0	}
142	0	if (blocksize < 32 * 1024) {
143		/* Do not use a too small blocksize (< 32 KB) when typesize is small */
144	0	blocksize = 32 * 1024;
145	0	}
146	0	}
147
148	0	last:
149		/* Check that blocksize is not too large */
150	0	if (blocksize > nbytes) {
151	0	blocksize = nbytes;
152	0	}
153
154		// blocksize must absolutely be a multiple of the typesize
155	0	if (blocksize > typesize) {
156	0	blocksize = blocksize / typesize * typesize;
157	0	}
158
159	0	context->blocksize = blocksize;
160	0	BLOSC_INFO("compcode: %d, clevel: %d, blocksize: %d, splitmode: %d, typesize: %d",
161	0	context->compcode, context->clevel, blocksize, splitmode, typesize);
162
163	0	return BLOSC2_ERROR_SUCCESS;
164	0	}
165
166	0	int blosc_stune_next_cparams(blosc2_context * context) {
167	0	BLOSC_UNUSED_PARAM(context);
168
169	0	return BLOSC2_ERROR_SUCCESS;
170	0	}
171
172	0	int blosc_stune_update(blosc2_context * context, double ctime) {
173	0	BLOSC_UNUSED_PARAM(context);
174	0	BLOSC_UNUSED_PARAM(ctime);
175
176	0	return BLOSC2_ERROR_SUCCESS;
177	0	}
178
179	0	int blosc_stune_free(blosc2_context * context) {
180	0	BLOSC_UNUSED_PARAM(context);
181
182	0	return BLOSC2_ERROR_SUCCESS;
183	0	}
184
185	0	int split_block(blosc2_context *context, int32_t typesize, int32_t blocksize) {
186	0	switch (context->splitmode) {
187	0	case BLOSC_ALWAYS_SPLIT:
188	0	return 1;
189	0	case BLOSC_NEVER_SPLIT:
190	0	return 0;
191	0	case BLOSC_FORWARD_COMPAT_SPLIT:
192	0	case BLOSC_AUTO_SPLIT:
193		// These cases will be handled later
194	0	break;
195	0	default:
196	0	BLOSC_TRACE_WARNING("Unrecognized split mode. Default to BLOSC_FORWARD_COMPAT_SPLIT");
197	0	}
198
199	0	int compcode = context->compcode;
200	0	return (
201		// Fast codecs like blosclz, lz4 seems to prefer to split
202	0	((compcode == BLOSC_BLOSCLZ) \|\| (compcode == BLOSC_LZ4)
203		// and low levels of zstd too
204	0	\|\| ((compcode == BLOSC_ZSTD) && (context->clevel <= 5))
205	0	) &&
206		// ...but split seems to harm cratio too much when not using shuffle
207	0	(context->filter_flags & BLOSC_DOSHUFFLE) &&
208	0	(typesize <= MAX_STREAMS) &&
209	0	(blocksize / typesize) >= BLOSC_MIN_BUFFERSIZE);
210	0	}