/src/c-blosc2/blosc/stune.c

Source
/*********************************************************************
  Blosc - Blocked Shuffling and Compression Library

  Copyright (c) 2021  Blosc Development Team <blosc@blosc.org>
  https://blosc.org
  License: BSD 3-Clause (see LICENSE.txt)

  See LICENSE.txt for details about copyright and rights to use.
**********************************************************************/

#include "stune.h"

#include <stdbool.h>
#include <stdio.h>


/* Whether a codec is meant for High Compression Ratios
   Includes LZ4 + BITSHUFFLE here, but not BloscLZ + BITSHUFFLE because,
   for some reason, the latter does not work too well */
static bool is_HCR(blosc2_context * context) {
  switch (context->compcode) {
    case BLOSC_BLOSCLZ :
      return false;
    case BLOSC_LZ4 :
      // return (context->filter_flags & BLOSC_DOBITSHUFFLE) ? true : false;
      // Do not treat LZ4 differently than BloscLZ here
      return false;
    case BLOSC_LZ4HC :
    case BLOSC_ZLIB :
    case BLOSC_ZSTD :
      return true;
    default :
      return false;
  }
}

int blosc_stune_init(void * config, blosc2_context* cctx, blosc2_context* dctx) {
  BLOSC_UNUSED_PARAM(config);
  BLOSC_UNUSED_PARAM(cctx);
  BLOSC_UNUSED_PARAM(dctx);

  return BLOSC2_ERROR_SUCCESS;
}

// Set the automatic blocksize 0 to its real value
int blosc_stune_next_blocksize(blosc2_context *context) {
  int32_t clevel = context->clevel;
  int32_t typesize = context->typesize;
  int32_t nbytes = context->sourcesize;
  int32_t user_blocksize = context->blocksize;
  int32_t blocksize = nbytes;

  // Protection against very small buffers
  if (nbytes < typesize) {
    context->blocksize = 1;
    return BLOSC2_ERROR_SUCCESS;
  }

  int splitmode = split_block(context, typesize, blocksize);
  if (user_blocksize) {
    blocksize = user_blocksize;
    goto last;
  }

  if (nbytes >= L1) {
    blocksize = L1;

    /* For HCR codecs, increase the block sizes by a factor of 2 because they
        are meant for compressing large blocks (i.e. they show a big overhead
        when compressing small ones). */
    if (is_HCR(context)) {
      blocksize *= 2;
    }

    // Choose a different blocksize depending on the compression level
    switch (clevel) {
      case 0:
        // Case of plain copy
        blocksize /= 4;
        break;
      case 1:
        blocksize /= 2;
        break;
      case 2:
        blocksize *= 1;
        break;
      case 3:
        blocksize *= 2;
        break;
      case 4:
      case 5:
        blocksize *= 4;
        break;
      case 6:
      case 7:
      case 8:
        blocksize *= 8;
        break;
      case 9:
        // Do not exceed 256 KB for non HCR codecs
        blocksize *= 8;
        if (is_HCR(context)) {
          blocksize *= 2;
        }
        break;
      default:
        break;
    }
  }

  /* Now the blocksize for splittable codecs */
  if (clevel > 0 && splitmode) {
    // For performance reasons, do not exceed 256 KB (it must fit in L2 cache)
    switch (clevel) {
      case 1:
      case 2:
      case 3:
        blocksize = 32 * 1024;
        break;
      case 4:
      case 5:
      case 6:
        blocksize = 64 * 1024;
        break;
      case 7:
        blocksize = 128 * 1024;
        break;
      case 8:
        blocksize = 256 * 1024;
        break;
      case 9:
      default:
        blocksize = 512 * 1024;
        break;
    }
    // Multiply by typesize to get proper split sizes
    blocksize *= typesize;
    // But do not exceed 4 MB per thread (having this capacity in L3 is normal in modern CPUs)
    if (blocksize > 4 * 1024 * 1024) {
      blocksize = 4 * 1024 * 1024;
    }
    if (blocksize < 32 * 1024) {
      /* Do not use a too small blocksize (< 32 KB) when typesize is small */
      blocksize = 32 * 1024;
    }
  }

  last:
  /* Check that blocksize is not too large */
  if (blocksize > nbytes) {
    blocksize = nbytes;
  }

  // blocksize *must absolutely* be a multiple of the typesize
  if (blocksize > typesize) {
    blocksize = blocksize / typesize * typesize;
  }

  context->blocksize = blocksize;
  BLOSC_INFO("compcode: %d, clevel: %d, blocksize: %d, splitmode: %d, typesize: %d",
             context->compcode, context->clevel, blocksize, splitmode, typesize);

  return BLOSC2_ERROR_SUCCESS;
}

int blosc_stune_next_cparams(blosc2_context * context) {
  BLOSC_UNUSED_PARAM(context);

  return BLOSC2_ERROR_SUCCESS;
}

int blosc_stune_update(blosc2_context * context, double ctime) {
  BLOSC_UNUSED_PARAM(context);
  BLOSC_UNUSED_PARAM(ctime);

  return BLOSC2_ERROR_SUCCESS;
}

int blosc_stune_free(blosc2_context * context) {
  BLOSC_UNUSED_PARAM(context);

  return BLOSC2_ERROR_SUCCESS;
}

int split_block(blosc2_context *context, int32_t typesize, int32_t blocksize) {
  switch (context->splitmode) {
    case BLOSC_ALWAYS_SPLIT:
      return 1;
    case BLOSC_NEVER_SPLIT:
      return 0;
    case BLOSC_FORWARD_COMPAT_SPLIT:
    case BLOSC_AUTO_SPLIT:
      // These cases will be handled later
      break;
    default:
      BLOSC_TRACE_WARNING("Unrecognized split mode.  Default to BLOSC_FORWARD_COMPAT_SPLIT");
  }

  int compcode = context->compcode;
  return (
          // Fast codecs like blosclz, lz4 seems to prefer to split
          ((compcode == BLOSC_BLOSCLZ) || (compcode == BLOSC_LZ4)
            // and low levels of zstd too
            || ((compcode == BLOSC_ZSTD) && (context->clevel <= 5))
            ) &&
          // ...but split seems to harm cratio too much when not using shuffle
          (context->filter_flags & BLOSC_DOSHUFFLE) &&
          (typesize <= MAX_STREAMS) &&
          (blocksize / typesize) >= BLOSC_MIN_BUFFERSIZE);
}

Coverage Report

Created: 2025-12-14 06:03

Line	Count	Source
1		/*********************************************************************
2		Blosc - Blocked Shuffling and Compression Library
3
4		Copyright (c) 2021 Blosc Development Team <blosc@blosc.org>
5		https://blosc.org
6		License: BSD 3-Clause (see LICENSE.txt)
7
8		See LICENSE.txt for details about copyright and rights to use.
9		**********************************************************************/
10
11		#include "stune.h"
12
13		#include <stdbool.h>
14		#include <stdio.h>
15
16
17		/* Whether a codec is meant for High Compression Ratios
18		Includes LZ4 + BITSHUFFLE here, but not BloscLZ + BITSHUFFLE because,
19		for some reason, the latter does not work too well */
20	346	static bool is_HCR(blosc2_context * context) {
21	346	switch (context->compcode) {
22	346	case BLOSC_BLOSCLZ :
23	346	return false;
24	0	case BLOSC_LZ4 :
25		// return (context->filter_flags & BLOSC_DOBITSHUFFLE) ? true : false;
26		// Do not treat LZ4 differently than BloscLZ here
27	0	return false;
28	0	case BLOSC_LZ4HC :
29	0	case BLOSC_ZLIB :
30	0	case BLOSC_ZSTD :
31	0	return true;
32	0	default :
33	0	return false;
34	346	}
35	346	}
36
37	0	int blosc_stune_init(void * config, blosc2_context* cctx, blosc2_context* dctx) {
38	0	BLOSC_UNUSED_PARAM(config);
39	0	BLOSC_UNUSED_PARAM(cctx);
40	0	BLOSC_UNUSED_PARAM(dctx);
41
42	0	return BLOSC2_ERROR_SUCCESS;
43	0	}
44
45		// Set the automatic blocksize 0 to its real value
46	38.1k	int blosc_stune_next_blocksize(blosc2_context *context) {
47	38.1k	int32_t clevel = context->clevel;
48	38.1k	int32_t typesize = context->typesize;
49	38.1k	int32_t nbytes = context->sourcesize;
50	38.1k	int32_t user_blocksize = context->blocksize;
51	38.1k	int32_t blocksize = nbytes;
52
53		// Protection against very small buffers
54	38.1k	if (nbytes < typesize) {
55	0	context->blocksize = 1;
56	0	return BLOSC2_ERROR_SUCCESS;
57	0	}
58
59	38.1k	int splitmode = split_block(context, typesize, blocksize);
60	38.1k	if (user_blocksize) {
61	36.0k	blocksize = user_blocksize;
62	36.0k	goto last;
63	36.0k	}
64
65	2.14k	if (nbytes >= L1) {
66	312	blocksize = L1;
67
68		/* For HCR codecs, increase the block sizes by a factor of 2 because they
69		are meant for compressing large blocks (i.e. they show a big overhead
70		when compressing small ones). */
71	312	if (is_HCR(context)) {
72	0	blocksize *= 2;
73	0	}
74
75		// Choose a different blocksize depending on the compression level
76	312	switch (clevel) {
77	40	case 0:
78		// Case of plain copy
79	40	blocksize /= 4;
80	40	break;
81	111	case 1:
82	111	blocksize /= 2;
83	111	break;
84	19	case 2:
85	19	blocksize *= 1;
86	19	break;
87	9	case 3:
88	9	blocksize *= 2;
89	9	break;
90	3	case 4:
91	66	case 5:
92	66	blocksize *= 4;
93	66	break;
94	1	case 6:
95	31	case 7:
96	33	case 8:
97	33	blocksize *= 8;
98	33	break;
99	34	case 9:
100		// Do not exceed 256 KB for non HCR codecs
101	34	blocksize *= 8;
102	34	if (is_HCR(context)) {
103	0	blocksize *= 2;
104	0	}
105	34	break;
106	0	default:
107	0	break;
108	312	}
109	312	}
110
111		/* Now the blocksize for splittable codecs */
112	2.14k	if (clevel > 0 && splitmode) {
113		// For performance reasons, do not exceed 256 KB (it must fit in L2 cache)
114	966	switch (clevel) {
115	293	case 1:
116	314	case 2:
117	328	case 3:
118	328	blocksize = 32 * 1024;
119	328	break;
120	22	case 4:
121	50	case 5:
122	60	case 6:
123	60	blocksize = 64 * 1024;
124	60	break;
125	164	case 7:
126	164	blocksize = 128 * 1024;
127	164	break;
128	91	case 8:
129	91	blocksize = 256 * 1024;
130	91	break;
131	323	case 9:
132	323	default:
133	323	blocksize = 512 * 1024;
134	323	break;
135	966	}
136		// Multiply by typesize to get proper split sizes
137	966	blocksize *= typesize;
138		// But do not exceed 4 MB per thread (having this capacity in L3 is normal in modern CPUs)
139	966	if (blocksize > 4 * 1024 * 1024) {
140	0	blocksize = 4 * 1024 * 1024;
141	0	}
142	966	if (blocksize < 32 * 1024) {
143		/* Do not use a too small blocksize (< 32 KB) when typesize is small */
144	0	blocksize = 32 * 1024;
145	0	}
146	966	}
147
148	38.1k	last:
149		/* Check that blocksize is not too large */
150	38.1k	if (blocksize > nbytes) {
151	1.34k	blocksize = nbytes;
152	1.34k	}
153
154		// blocksize must absolutely be a multiple of the typesize
155	38.1k	if (blocksize > typesize) {
156	38.0k	blocksize = blocksize / typesize * typesize;
157	38.0k	}
158
159	38.1k	context->blocksize = blocksize;
160	38.1k	BLOSC_INFO("compcode: %d, clevel: %d, blocksize: %d, splitmode: %d, typesize: %d",
161	38.1k	context->compcode, context->clevel, blocksize, splitmode, typesize);
162
163	38.1k	return BLOSC2_ERROR_SUCCESS;
164	2.14k	}
165
166	0	int blosc_stune_next_cparams(blosc2_context * context) {
167	0	BLOSC_UNUSED_PARAM(context);
168
169	0	return BLOSC2_ERROR_SUCCESS;
170	0	}
171
172	0	int blosc_stune_update(blosc2_context * context, double ctime) {
173	0	BLOSC_UNUSED_PARAM(context);
174	0	BLOSC_UNUSED_PARAM(ctime);
175
176	0	return BLOSC2_ERROR_SUCCESS;
177	0	}
178
179	0	int blosc_stune_free(blosc2_context * context) {
180	0	BLOSC_UNUSED_PARAM(context);
181
182	0	return BLOSC2_ERROR_SUCCESS;
183	0	}
184
185	75.8k	int split_block(blosc2_context *context, int32_t typesize, int32_t blocksize) {
186	75.8k	switch (context->splitmode) {
187	0	case BLOSC_ALWAYS_SPLIT:
188	0	return 1;
189	0	case BLOSC_NEVER_SPLIT:
190	0	return 0;
191	75.8k	case BLOSC_FORWARD_COMPAT_SPLIT:
192	75.8k	case BLOSC_AUTO_SPLIT:
193		// These cases will be handled later
194	75.8k	break;
195	0	default:
196	0	BLOSC_TRACE_WARNING("Unrecognized split mode. Default to BLOSC_FORWARD_COMPAT_SPLIT");
197	75.8k	}
198
199	75.8k	int compcode = context->compcode;
200	75.8k	return (
201		// Fast codecs like blosclz, lz4 seems to prefer to split
202	75.8k	((compcode == BLOSC_BLOSCLZ) \|\| (compcode == BLOSC_LZ4)
203		// and low levels of zstd too
204	0	\|\| ((compcode == BLOSC_ZSTD) && (context->clevel <= 5))
205	75.8k	) &&
206		// ...but split seems to harm cratio too much when not using shuffle
207	75.8k	(context->filter_flags & BLOSC_DOSHUFFLE) &&
208	20.3k	(typesize <= MAX_STREAMS) &&
209	20.3k	(blocksize / typesize) >= BLOSC_MIN_BUFFERSIZE);
210	75.8k	}