/src/c-blosc2/blosc/stune.c

Source
/*********************************************************************
  Blosc - Blocked Shuffling and Compression Library

  Copyright (c) 2021  Blosc Development Team <blosc@blosc.org>
  https://blosc.org
  License: BSD 3-Clause (see LICENSE.txt)

  See LICENSE.txt for details about copyright and rights to use.
**********************************************************************/

#include "stune.h"

#include <stdbool.h>
#include <stdio.h>


/* Whether a codec is meant for High Compression Ratios
   Includes LZ4 + BITSHUFFLE here, but not BloscLZ + BITSHUFFLE because,
   for some reason, the latter does not work too well */
static bool is_HCR(blosc2_context * context) {
  switch (context->compcode) {
    case BLOSC_BLOSCLZ :
      return false;
    case BLOSC_LZ4 :
      // return (context->filter_flags & BLOSC_DOBITSHUFFLE) ? true : false;
      // Do not treat LZ4 differently than BloscLZ here
      return false;
    case BLOSC_LZ4HC :
    case BLOSC_ZLIB :
    case BLOSC_ZSTD :
      return true;
    default :
      return false;
  }
}

int blosc_stune_init(void * config, blosc2_context* cctx, blosc2_context* dctx) {
  BLOSC_UNUSED_PARAM(config);
  BLOSC_UNUSED_PARAM(cctx);
  BLOSC_UNUSED_PARAM(dctx);

  return BLOSC2_ERROR_SUCCESS;
}

// Set the automatic blocksize 0 to its real value
int blosc_stune_next_blocksize(blosc2_context *context) {
  int32_t clevel = context->clevel;
  int32_t typesize = context->typesize;
  int32_t nbytes = context->sourcesize;
  int32_t user_blocksize = context->blocksize;
  int32_t blocksize = nbytes;

  // Protection against very small buffers
  if (nbytes < typesize) {
    context->blocksize = 1;
    return BLOSC2_ERROR_SUCCESS;
  }

  int splitmode = split_block(context, typesize, blocksize);
  if (user_blocksize) {
    blocksize = user_blocksize;
    goto last;
  }

  if (nbytes >= L1) {
    blocksize = L1;

    /* For HCR codecs, increase the block sizes by a factor of 2 because they
        are meant for compressing large blocks (i.e. they show a big overhead
        when compressing small ones). */
    if (is_HCR(context)) {
      blocksize *= 2;
    }

    // Choose a different blocksize depending on the compression level
    switch (clevel) {
      case 0:
        // Case of plain copy
        blocksize /= 4;
        break;
      case 1:
        blocksize /= 2;
        break;
      case 2:
        blocksize *= 1;
        break;
      case 3:
        blocksize *= 2;
        break;
      case 4:
      case 5:
        blocksize *= 4;
        break;
      case 6:
      case 7:
      case 8:
        blocksize *= 8;
        break;
      case 9:
        // Do not exceed 256 KB for non HCR codecs
        blocksize *= 8;
        if (is_HCR(context)) {
          blocksize *= 2;
        }
        break;
      default:
        break;
    }
  }

  /* Now the blocksize for splittable codecs */
  if (clevel > 0 && splitmode) {
    // For performance reasons, do not exceed 256 KB (it must fit in L2 cache)
    switch (clevel) {
      case 1:
      case 2:
      case 3:
        blocksize = 32 * 1024;
        break;
      case 4:
      case 5:
      case 6:
        blocksize = 64 * 1024;
        break;
      case 7:
        blocksize = 128 * 1024;
        break;
      case 8:
        blocksize = 256 * 1024;
        break;
      case 9:
      default:
        blocksize = 512 * 1024;
        break;
    }
    // Multiply by typesize to get proper split sizes
    blocksize *= typesize;
    // But do not exceed 4 MB per thread (having this capacity in L3 is normal in modern CPUs)
    if (blocksize > 4 * 1024 * 1024) {
      blocksize = 4 * 1024 * 1024;
    }
    if (blocksize < 32 * 1024) {
      /* Do not use a too small blocksize (< 32 KB) when typesize is small */
      blocksize = 32 * 1024;
    }
  }

  last:
  /* Check that blocksize is not too large */
  if (blocksize > nbytes) {
    blocksize = nbytes;
  }

  // blocksize *must absolutely* be a multiple of the typesize
  if (blocksize > typesize) {
    blocksize = blocksize / typesize * typesize;
  }

  context->blocksize = blocksize;
  BLOSC_INFO("compcode: %d, clevel: %d, blocksize: %d, splitmode: %d, typesize: %d",
             context->compcode, context->clevel, blocksize, splitmode, typesize);

  return BLOSC2_ERROR_SUCCESS;
}

int blosc_stune_next_cparams(blosc2_context * context) {
  BLOSC_UNUSED_PARAM(context);

  return BLOSC2_ERROR_SUCCESS;
}

int blosc_stune_update(blosc2_context * context, double ctime) {
  BLOSC_UNUSED_PARAM(context);
  BLOSC_UNUSED_PARAM(ctime);

  return BLOSC2_ERROR_SUCCESS;
}

int blosc_stune_free(blosc2_context * context) {
  BLOSC_UNUSED_PARAM(context);

  return BLOSC2_ERROR_SUCCESS;
}

int split_block(blosc2_context *context, int32_t typesize, int32_t blocksize) {
  switch (context->splitmode) {
    case BLOSC_ALWAYS_SPLIT:
      return 1;
    case BLOSC_NEVER_SPLIT:
      return 0;
    case BLOSC_FORWARD_COMPAT_SPLIT:
    case BLOSC_AUTO_SPLIT:
      // These cases will be handled later
      break;
    default:
      BLOSC_TRACE_WARNING("Unrecognized split mode.  Default to BLOSC_FORWARD_COMPAT_SPLIT");
  }

  int compcode = context->compcode;
  return (
          // Fast codecs like blosclz, lz4 seems to prefer to split
          ((compcode == BLOSC_BLOSCLZ) || (compcode == BLOSC_LZ4)
            // and low levels of zstd too
            || ((compcode == BLOSC_ZSTD) && (context->clevel <= 5))
            ) &&
          // ...but split seems to harm cratio too much when not using shuffle
          (context->filter_flags & BLOSC_DOSHUFFLE) &&
          (typesize <= MAX_STREAMS) &&
          (blocksize / typesize) >= BLOSC_MIN_BUFFERSIZE);
}

Coverage Report

Created: 2025-12-24 07:17

Line	Count	Source
1		/*********************************************************************
2		Blosc - Blocked Shuffling and Compression Library
3
4		Copyright (c) 2021 Blosc Development Team <blosc@blosc.org>
5		https://blosc.org
6		License: BSD 3-Clause (see LICENSE.txt)
7
8		See LICENSE.txt for details about copyright and rights to use.
9		**********************************************************************/
10
11		#include "stune.h"
12
13		#include <stdbool.h>
14		#include <stdio.h>
15
16
17		/* Whether a codec is meant for High Compression Ratios
18		Includes LZ4 + BITSHUFFLE here, but not BloscLZ + BITSHUFFLE because,
19		for some reason, the latter does not work too well */
20	363	static bool is_HCR(blosc2_context * context) {
21	363	switch (context->compcode) {
22	363	case BLOSC_BLOSCLZ :
23	363	return false;
24	0	case BLOSC_LZ4 :
25		// return (context->filter_flags & BLOSC_DOBITSHUFFLE) ? true : false;
26		// Do not treat LZ4 differently than BloscLZ here
27	0	return false;
28	0	case BLOSC_LZ4HC :
29	0	case BLOSC_ZLIB :
30	0	case BLOSC_ZSTD :
31	0	return true;
32	0	default :
33	0	return false;
34	363	}
35	363	}
36
37	0	int blosc_stune_init(void * config, blosc2_context* cctx, blosc2_context* dctx) {
38	0	BLOSC_UNUSED_PARAM(config);
39	0	BLOSC_UNUSED_PARAM(cctx);
40	0	BLOSC_UNUSED_PARAM(dctx);
41
42	0	return BLOSC2_ERROR_SUCCESS;
43	0	}
44
45		// Set the automatic blocksize 0 to its real value
46	52.9k	int blosc_stune_next_blocksize(blosc2_context *context) {
47	52.9k	int32_t clevel = context->clevel;
48	52.9k	int32_t typesize = context->typesize;
49	52.9k	int32_t nbytes = context->sourcesize;
50	52.9k	int32_t user_blocksize = context->blocksize;
51	52.9k	int32_t blocksize = nbytes;
52
53		// Protection against very small buffers
54	52.9k	if (nbytes < typesize) {
55	0	context->blocksize = 1;
56	0	return BLOSC2_ERROR_SUCCESS;
57	0	}
58
59	52.9k	int splitmode = split_block(context, typesize, blocksize);
60	52.9k	if (user_blocksize) {
61	50.5k	blocksize = user_blocksize;
62	50.5k	goto last;
63	50.5k	}
64
65	2.36k	if (nbytes >= L1) {
66	326	blocksize = L1;
67
68		/* For HCR codecs, increase the block sizes by a factor of 2 because they
69		are meant for compressing large blocks (i.e. they show a big overhead
70		when compressing small ones). */
71	326	if (is_HCR(context)) {
72	0	blocksize *= 2;
73	0	}
74
75		// Choose a different blocksize depending on the compression level
76	326	switch (clevel) {
77	39	case 0:
78		// Case of plain copy
79	39	blocksize /= 4;
80	39	break;
81	118	case 1:
82	118	blocksize /= 2;
83	118	break;
84	16	case 2:
85	16	blocksize *= 1;
86	16	break;
87	13	case 3:
88	13	blocksize *= 2;
89	13	break;
90	3	case 4:
91	64	case 5:
92	64	blocksize *= 4;
93	64	break;
94	1	case 6:
95	35	case 7:
96	39	case 8:
97	39	blocksize *= 8;
98	39	break;
99	37	case 9:
100		// Do not exceed 256 KB for non HCR codecs
101	37	blocksize *= 8;
102	37	if (is_HCR(context)) {
103	0	blocksize *= 2;
104	0	}
105	37	break;
106	0	default:
107	0	break;
108	326	}
109	326	}
110
111		/* Now the blocksize for splittable codecs */
112	2.36k	if (clevel > 0 && splitmode) {
113		// For performance reasons, do not exceed 256 KB (it must fit in L2 cache)
114	1.06k	switch (clevel) {
115	303	case 1:
116	322	case 2:
117	350	case 3:
118	350	blocksize = 32 * 1024;
119	350	break;
120	23	case 4:
121	51	case 5:
122	57	case 6:
123	57	blocksize = 64 * 1024;
124	57	break;
125	230	case 7:
126	230	blocksize = 128 * 1024;
127	230	break;
128	87	case 8:
129	87	blocksize = 256 * 1024;
130	87	break;
131	336	case 9:
132	336	default:
133	336	blocksize = 512 * 1024;
134	336	break;
135	1.06k	}
136		// Multiply by typesize to get proper split sizes
137	1.06k	blocksize *= typesize;
138		// But do not exceed 4 MB per thread (having this capacity in L3 is normal in modern CPUs)
139	1.06k	if (blocksize > 4 * 1024 * 1024) {
140	0	blocksize = 4 * 1024 * 1024;
141	0	}
142	1.06k	if (blocksize < 32 * 1024) {
143		/* Do not use a too small blocksize (< 32 KB) when typesize is small */
144	0	blocksize = 32 * 1024;
145	0	}
146	1.06k	}
147
148	52.9k	last:
149		/* Check that blocksize is not too large */
150	52.9k	if (blocksize > nbytes) {
151	9.28k	blocksize = nbytes;
152	9.28k	}
153
154		// blocksize must absolutely be a multiple of the typesize
155	52.9k	if (blocksize > typesize) {
156	52.8k	blocksize = blocksize / typesize * typesize;
157	52.8k	}
158
159	52.9k	context->blocksize = blocksize;
160	52.9k	BLOSC_INFO("compcode: %d, clevel: %d, blocksize: %d, splitmode: %d, typesize: %d",
161	52.9k	context->compcode, context->clevel, blocksize, splitmode, typesize);
162
163	52.9k	return BLOSC2_ERROR_SUCCESS;
164	2.36k	}
165
166	0	int blosc_stune_next_cparams(blosc2_context * context) {
167	0	BLOSC_UNUSED_PARAM(context);
168
169	0	return BLOSC2_ERROR_SUCCESS;
170	0	}
171
172	0	int blosc_stune_update(blosc2_context * context, double ctime) {
173	0	BLOSC_UNUSED_PARAM(context);
174	0	BLOSC_UNUSED_PARAM(ctime);
175
176	0	return BLOSC2_ERROR_SUCCESS;
177	0	}
178
179	0	int blosc_stune_free(blosc2_context * context) {
180	0	BLOSC_UNUSED_PARAM(context);
181
182	0	return BLOSC2_ERROR_SUCCESS;
183	0	}
184
185	105k	int split_block(blosc2_context *context, int32_t typesize, int32_t blocksize) {
186	105k	switch (context->splitmode) {
187	0	case BLOSC_ALWAYS_SPLIT:
188	0	return 1;
189	0	case BLOSC_NEVER_SPLIT:
190	0	return 0;
191	105k	case BLOSC_FORWARD_COMPAT_SPLIT:
192	105k	case BLOSC_AUTO_SPLIT:
193		// These cases will be handled later
194	105k	break;
195	0	default:
196	0	BLOSC_TRACE_WARNING("Unrecognized split mode. Default to BLOSC_FORWARD_COMPAT_SPLIT");
197	105k	}
198
199	105k	int compcode = context->compcode;
200	105k	return (
201		// Fast codecs like blosclz, lz4 seems to prefer to split
202	105k	((compcode == BLOSC_BLOSCLZ) \|\| (compcode == BLOSC_LZ4)
203		// and low levels of zstd too
204	23.6k	\|\| ((compcode == BLOSC_ZSTD) && (context->clevel <= 5))
205	105k	) &&
206		// ...but split seems to harm cratio too much when not using shuffle
207	89.7k	(context->filter_flags & BLOSC_DOSHUFFLE) &&
208	25.5k	(typesize <= MAX_STREAMS) &&
209	25.5k	(blocksize / typesize) >= BLOSC_MIN_BUFFERSIZE);
210	105k	}