Coverage Report

Created: 2026-10-28 07:09

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/ffmpeg/libswscale/ops_dispatch.c
Line
Count
Source
1
/**
2
 * Copyright (C) 2025 Niklas Haas
3
 *
4
 * This file is part of FFmpeg.
5
 *
6
 * FFmpeg is free software; you can redistribute it and/or
7
 * modify it under the terms of the GNU Lesser General Public
8
 * License as published by the Free Software Foundation; either
9
 * version 2.1 of the License, or (at your option) any later version.
10
 *
11
 * FFmpeg is distributed in the hope that it will be useful,
12
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
 * Lesser General Public License for more details.
15
 *
16
 * You should have received a copy of the GNU Lesser General Public
17
 * License along with FFmpeg; if not, write to the Free Software
18
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19
 */
20
21
#include "libavutil/avassert.h"
22
#include "libavutil/cpu.h"
23
#include "libavutil/mathematics.h"
24
#include "libavutil/mem.h"
25
#include "libavutil/mem_internal.h"
26
#include "libavutil/refstruct.h"
27
28
#include "ops.h"
29
#include "ops_internal.h"
30
#include "ops_dispatch.h"
31
#include "swscale_internal.h"
32
33
#define RET(x)                                                                 \
34
0
    do {                                                                       \
35
0
        if ((ret = (x)) < 0)                                                   \
36
0
            goto fail;                                                         \
37
0
    } while (0)
38
39
typedef struct SwsOpPass {
40
    SwsCompiledOp comp;
41
    SwsOpExec exec_base;
42
    SwsOpExec exec_tail;
43
    size_t num_blocks;
44
    int tail_off_in;
45
    int tail_off_out;
46
    int tail_size_in;
47
    int tail_size_out;
48
    int planes_in;
49
    int planes_out;
50
    int pixel_bits_in;
51
    int pixel_bits_out;
52
    int idx_in[4];
53
    int idx_out[4];
54
    int palette_idx;
55
    int *offsets_y;
56
    int filter_size_h;
57
    int filter_size_v;
58
    bool memcpy_in;
59
    bool memcpy_out;
60
    size_t tail_blocks;
61
    uint8_t *tail_buf; /* extra memory for fixing unpadded tails */
62
    unsigned int tail_buf_size;
63
} SwsOpPass;
64
65
static int compile_backend(SwsContext *ctx, const SwsOpBackend *backend,
66
                           const SwsOpList *ops, SwsCompiledOp *out)
67
0
{
68
0
    SwsOpList *copy;
69
0
    SwsCompiledOp compiled = {0};
70
0
    int ret = 0;
71
72
0
    copy = ff_sws_op_list_duplicate(ops);
73
0
    if (!copy)
74
0
        return AVERROR(ENOMEM);
75
76
    /* Ensure these are always set during compilation */
77
0
    ff_sws_op_list_update(copy);
78
79
0
    ret = backend->compile(ctx, copy, &compiled);
80
0
    if (ret < 0) {
81
0
        int msg_lev = ret == AVERROR(ENOTSUP) ? AV_LOG_TRACE : AV_LOG_ERROR;
82
0
        av_log(ctx, msg_lev, "Backend '%s' failed to compile operations: %s\n",
83
0
               backend->name, av_err2str(ret));
84
0
        goto fail;
85
0
    }
86
87
0
    compiled.backend = backend;
88
0
    *out = compiled;
89
90
0
    av_log(ctx, AV_LOG_VERBOSE, "Compiled using backend '%s': "
91
0
           "block size = %d, over-read = {%d %d %d %d}, over-write = {%d %d %d %d}, "
92
0
           "cpu flags = 0x%x\n", backend->name, out->block_size,
93
0
           out->over_read[0], out->over_read[1],
94
0
           out->over_read[2], out->over_read[3],
95
0
           out->over_write[0], out->over_write[1],
96
0
           out->over_write[2], out->over_write[3],
97
0
           out->cpu_flags);
98
99
0
    ff_sws_op_list_print(ctx, AV_LOG_VERBOSE, AV_LOG_TRACE, ops);
100
101
0
fail:
102
0
    ff_sws_op_list_free(&copy);
103
0
    return ret;
104
0
}
105
106
int ff_sws_ops_compile(SwsContext *ctx, const SwsOpBackend *backend,
107
                       const SwsOpList *ops, SwsCompiledOp *out)
108
0
{
109
0
    if (backend)
110
0
        return compile_backend(ctx, backend, ops, out);
111
112
0
    const SwsBackend enabled = ff_sws_enabled_backends(ctx);
113
0
    for (int n = 0; ff_sws_op_backends[n]; n++) {
114
0
        const SwsOpBackend *backend = ff_sws_op_backends[n];
115
0
        if (ops->src.hw_format != backend->hw_format ||
116
0
            ops->dst.hw_format != backend->hw_format ||
117
0
            !(enabled & backend->flags))
118
0
            continue;
119
0
        if (compile_backend(ctx, backend, ops, out) < 0)
120
0
            continue;
121
122
0
        return 0;
123
0
    }
124
125
0
    return AVERROR(ENOTSUP);
126
0
}
127
128
void ff_sws_compiled_op_unref(SwsCompiledOp *comp)
129
0
{
130
0
    if (comp->free)
131
0
        comp->free(comp->priv);
132
133
0
    *comp = (SwsCompiledOp) {0};
134
0
}
135
136
static void op_pass_free(void *ptr)
137
0
{
138
0
    SwsOpPass *p = ptr;
139
0
    if (!p)
140
0
        return;
141
142
0
    ff_sws_compiled_op_unref(&p->comp);
143
0
    av_refstruct_unref(&p->offsets_y);
144
0
    av_free(p->exec_base.in_bump_y);
145
0
    av_free(p->exec_base.in_offset_x);
146
0
    av_free(p->tail_buf);
147
0
    av_free(p);
148
0
}
149
150
static inline void get_row_data(const SwsOpPass *p, const int y_dst,
151
                                const uint8_t *in[4], uint8_t *out[4])
152
0
{
153
0
    const SwsOpExec *base = &p->exec_base;
154
0
    const int y_src = p->offsets_y ? p->offsets_y[y_dst] : y_dst;
155
0
    for (int i = 0; i < p->planes_in; i++)
156
0
        in[i] = base->in[i] + (y_src >> base->in_sub_y[i]) * base->in_stride[i];
157
0
    for (int i = 0; i < p->planes_out; i++)
158
0
        out[i] = base->out[i] + (y_dst >> base->out_sub_y[i]) * base->out_stride[i];
159
0
}
160
161
static inline size_t pixel_bytes(size_t pixels, int pixel_bits,
162
                                 enum AVRounding rounding)
163
0
{
164
0
    const uint64_t bits = (uint64_t) pixels * pixel_bits;
165
0
    switch (rounding) {
166
0
    case AV_ROUND_ZERO:
167
0
    case AV_ROUND_DOWN:
168
0
        return bits >> 3;
169
0
    case AV_ROUND_INF:
170
0
    case AV_ROUND_UP:
171
0
        return (bits + 7) >> 3;
172
0
    default:
173
0
        av_unreachable("Invalid rounding mode");
174
0
        return (size_t) -1;
175
0
    }
176
0
}
177
178
static size_t safe_bytes_pad(int linesize, int plane_pad)
179
0
{
180
0
    av_assert1(linesize);
181
0
    int64_t safe_bytes = FFABS((int64_t) linesize) - plane_pad;
182
0
    return FFMAX(safe_bytes, 0);
183
0
}
184
185
static size_t safe_blocks_offset(size_t num_blocks, unsigned block_size,
186
                                 ptrdiff_t safe_offset,
187
                                 const int32_t *offset_bytes)
188
0
{
189
0
    size_t safe_blocks = num_blocks;
190
0
    while (safe_blocks && offset_bytes[safe_blocks * block_size - 1] > safe_offset)
191
0
        safe_blocks--;
192
0
    return safe_blocks;
193
0
}
194
195
static void copy_lines(uint8_t *dst, const ptrdiff_t dst_stride,
196
                       const uint8_t *src, const ptrdiff_t src_stride,
197
                       const int h, const size_t bytes)
198
0
{
199
0
    for (int y = 0; y < h; y++) {
200
0
        memcpy(dst, src, bytes);
201
0
        dst += dst_stride;
202
0
        src += src_stride;
203
0
    }
204
0
}
205
206
static int op_pass_setup(const SwsFrame *out, const SwsFrame *in,
207
                         const SwsPass *pass)
208
0
{
209
0
    const AVPixFmtDescriptor *indesc  = av_pix_fmt_desc_get(in->format);
210
0
    const bool float_in = indesc->flags & AV_PIX_FMT_FLAG_FLOAT;
211
0
    const int width = out->width;
212
213
0
    SwsOpPass *p = pass->priv;
214
0
    SwsOpExec *exec = &p->exec_base;
215
0
    const SwsCompiledOp *comp = &p->comp;
216
217
    /* Set up main loop parameters */
218
0
    const unsigned block_size = comp->block_size;
219
0
    const size_t num_blocks   = p->num_blocks;
220
0
    const size_t aligned_w    = num_blocks * block_size;
221
0
    p->memcpy_in  = false;
222
0
    p->memcpy_out = false;
223
224
0
    size_t safe_blocks = num_blocks;
225
0
    for (int i = 0; i < p->planes_in; i++) {
226
0
        const int idx = p->idx_in[i];
227
0
        size_t input_bytes = in->linesize[idx];
228
0
        if (p->filter_size_h && float_in) {
229
            /* Floating point inputs may contain NaN / Infinity in the padding */
230
0
            const int plane_w = AV_CEIL_RSHIFT(in->width, exec->in_sub_x[i]);
231
0
            input_bytes = pixel_bytes(plane_w, p->pixel_bits_in, AV_ROUND_UP);
232
0
        }
233
234
0
        size_t safe_bytes = safe_bytes_pad(input_bytes, comp->over_read[i]);
235
0
        size_t safe_blocks_in;
236
0
        if (exec->in_offset_x) {
237
0
            size_t filter_size = pixel_bytes(p->filter_size_h, p->pixel_bits_in,
238
0
                                             AV_ROUND_UP);
239
0
            safe_blocks_in = safe_blocks_offset(num_blocks, block_size,
240
0
                                                safe_bytes - filter_size,
241
0
                                                exec->in_offset_x);
242
0
        } else {
243
0
            safe_blocks_in = safe_bytes / exec->block_size_in[i];
244
0
        }
245
246
0
        if (safe_blocks_in < num_blocks) {
247
0
            p->memcpy_in = true;
248
0
            safe_blocks  = FFMIN(safe_blocks, safe_blocks_in);
249
0
        }
250
251
0
        size_t loop_size   = num_blocks * exec->block_size_in[i];
252
0
        exec->in[i]        = in->data[idx];
253
0
        exec->in_stride[i] = in->linesize[idx];
254
0
        exec->in_bump[i]   = in->linesize[idx] - loop_size;
255
0
    }
256
257
0
    for (int i = 0; i < p->planes_out; i++) {
258
0
        const int idx = p->idx_out[i];
259
0
        size_t safe_bytes = safe_bytes_pad(out->linesize[idx], comp->over_write[i]);
260
0
        size_t safe_blocks_out = safe_bytes / exec->block_size_out[i];
261
0
        if (safe_blocks_out < num_blocks) {
262
0
            p->memcpy_out = true;
263
0
            safe_blocks   = FFMIN(safe_blocks, safe_blocks_out);
264
0
        }
265
266
0
        size_t loop_size    = num_blocks * exec->block_size_out[i];
267
0
        exec->out[i]        = out->data[idx];
268
0
        exec->out_stride[i] = out->linesize[idx];
269
0
        exec->out_bump[i]   = out->linesize[idx] - loop_size;
270
0
    }
271
272
0
    if (p->palette_idx >= 0) {
273
0
        exec->in[1] = in->data[p->palette_idx];
274
0
        exec->in_stride[1] = exec->in_bump[1] = 0;
275
0
    }
276
277
0
    if (!p->memcpy_in && !p->memcpy_out) {
278
0
        av_assert0(safe_blocks == num_blocks);
279
0
        return 0;
280
0
    }
281
282
    /* Set-up tail section parameters and buffers */
283
0
    SwsOpExec *tail = &p->exec_tail;
284
0
    const int align = av_cpu_max_align();
285
0
    size_t alloc_size = 0;
286
0
    *tail = *exec;
287
288
0
    const size_t safe_width = safe_blocks * block_size;
289
0
    const size_t tail_size  = width - safe_width;
290
0
    p->tail_off_out  = pixel_bytes(safe_width, p->pixel_bits_out, AV_ROUND_DOWN);
291
0
    p->tail_size_out = pixel_bytes(tail_size,  p->pixel_bits_out, AV_ROUND_UP);
292
0
    p->tail_blocks   = num_blocks - safe_blocks;
293
294
0
    if (exec->in_offset_x) {
295
0
        p->tail_off_in  = exec->in_offset_x[safe_width];
296
0
        p->tail_size_in = exec->in_offset_x[width - 1] - p->tail_off_in;
297
0
        p->tail_size_in += pixel_bytes(p->filter_size_h, p->pixel_bits_in, AV_ROUND_UP);
298
0
    } else {
299
0
        p->tail_off_in  = pixel_bytes(safe_width, p->pixel_bits_in, AV_ROUND_DOWN);
300
0
        p->tail_size_in = pixel_bytes(tail_size,  p->pixel_bits_in, AV_ROUND_UP);
301
0
    }
302
303
0
    const size_t alloc_width = aligned_w - safe_width;
304
0
    for (int i = 0; p->memcpy_in && i < p->planes_in; i++) {
305
0
        size_t needed_size;
306
0
        if (exec->in_offset_x) {
307
            /* The input offset map is already padded to multiples of the block
308
             * size, and clamps the input offsets to the image boundaries; so
309
             * we just need to compensate for the comp->over_read */
310
0
            needed_size = p->tail_size_in;
311
0
        } else {
312
0
            needed_size = pixel_bytes(alloc_width, p->pixel_bits_in, AV_ROUND_UP);
313
0
        }
314
0
        size_t loop_size   = p->tail_blocks * exec->block_size_in[i];
315
0
        tail->in_stride[i] = FFALIGN(needed_size + comp->over_read[i], align);
316
0
        tail->in_bump[i]   = tail->in_stride[i] - loop_size;
317
0
        alloc_size += tail->in_stride[i] * in->height;
318
0
    }
319
320
0
    for (int i = 0; p->memcpy_out && i < p->planes_out; i++) {
321
0
        size_t needed_size  = pixel_bytes(alloc_width, p->pixel_bits_out, AV_ROUND_UP);
322
0
        size_t loop_size    = p->tail_blocks * exec->block_size_out[i];
323
0
        tail->out_stride[i] = FFALIGN(needed_size + comp->over_write[i], align);
324
0
        tail->out_bump[i]   = tail->out_stride[i] - loop_size;
325
0
        alloc_size += tail->out_stride[i] * out->height;
326
0
    }
327
328
0
    if (p->memcpy_in && exec->in_offset_x) {
329
        /* `in_offset_x` is indexed relative to the line start, not the start
330
         * of the section being processed; so we need to over-allocate this
331
         * array to the full width of the image, even though we will only
332
         * partially fill in the offsets relevant to the tail region */
333
0
        alloc_size += aligned_w * sizeof(*exec->in_offset_x);
334
0
    }
335
336
0
    av_fast_mallocz(&p->tail_buf, &p->tail_buf_size, alloc_size);
337
0
    if (!p->tail_buf)
338
0
        return AVERROR(ENOMEM);
339
340
0
    uint8_t *tail_buf = p->tail_buf;
341
0
    for (int i = 0; p->memcpy_in && i < p->planes_in; i++) {
342
0
        tail->in[i] = tail_buf;
343
0
        tail_buf += tail->in_stride[i] * in->height;
344
0
    }
345
346
0
    for (int i = 0; p->memcpy_out && i < p->planes_out; i++) {
347
0
        tail->out[i] = tail_buf;
348
0
        tail_buf += tail->out_stride[i] * out->height;
349
0
    }
350
351
0
    if (p->memcpy_in && exec->in_offset_x) {
352
0
        tail->in_offset_x = (int32_t *) tail_buf;
353
0
        for (int i = safe_width; i < aligned_w; i++)
354
0
            tail->in_offset_x[i] = exec->in_offset_x[i] - p->tail_off_in;
355
0
    }
356
357
    /* If vertically filtering, we need to copy all lines before processing
358
     * any slice, because filters may read past the current slice bounds */
359
0
    for (int i = 0; p->memcpy_in && p->offsets_y && i < p->planes_in; i++) {
360
0
        const int lines = AV_CEIL_RSHIFT(in->height, exec->in_sub_y[i]);
361
0
        copy_lines((uint8_t *) tail->in[i], tail->in_stride[i],
362
0
                   exec->in[i] + p->tail_off_in, exec->in_stride[i],
363
0
                   lines, p->tail_size_in);
364
0
    }
365
366
0
    return 0;
367
0
}
368
369
static void op_pass_run(const SwsFrame *out, const SwsFrame *in, const int y,
370
                        const int h, const SwsPass *pass)
371
0
{
372
0
    const SwsOpPass *p = pass->priv;
373
0
    const SwsCompiledOp *comp = &p->comp;
374
375
    /* Fill exec metadata for this slice */
376
0
    DECLARE_ALIGNED_32(SwsOpExec, exec) = p->exec_base;
377
0
    exec.slice_y = y;
378
0
    exec.slice_h = h;
379
380
    /**
381
     *  To ensure safety, we need to consider the following:
382
     *
383
     * 1. We can overread the input, unless this is the last line of an
384
     *    unpadded buffer. All defined operations can handle arbitrary pixel
385
     *    input, so overread of arbitrary data is fine. For flipped images,
386
     *    this condition is actually *inverted* to where the first line is
387
     *    the one at the end of the buffer.
388
     *
389
     * 2. We can overwrite the output, as long as we don't write more than the
390
     *    amount of pixels that fit into one linesize. So we always need to
391
     *    memcpy the last column on the output side if unpadded.
392
     */
393
394
0
    const int y_in = p->offsets_y ? p->offsets_y[y] : y;
395
0
    const bool memcpy_in  = p->memcpy_in;
396
0
    const bool memcpy_out = p->memcpy_out;
397
0
    const size_t num_blocks  = p->num_blocks;
398
0
    const size_t tail_blocks = p->tail_blocks;
399
400
0
    get_row_data(p, y, exec.in, exec.out);
401
0
    if (!memcpy_in && !memcpy_out) {
402
        /* Fast path (fully aligned/padded inputs and outputs) */
403
0
        comp->func(&exec, comp->priv, 0, y, num_blocks, y + h);
404
0
        return;
405
0
    }
406
407
    /* Non-aligned case (slow path); process main blocks as normal, and
408
     * a separate tail (via memcpy into an appropriately padded buffer) */
409
0
    if (num_blocks > tail_blocks) {
410
0
        for (int i = 0; i < 4; i++) {
411
            /* We process fewer blocks, so the in_bump needs to be increased
412
             * to reflect that the plane pointers are left on the last block,
413
             * not the end of the processed line, after each loop iteration */
414
0
            exec.in_bump[i]  += exec.block_size_in[i]  * tail_blocks;
415
0
            exec.out_bump[i] += exec.block_size_out[i] * tail_blocks;
416
0
        }
417
418
0
        comp->func(&exec, comp->priv, 0, y, num_blocks - tail_blocks, y + h);
419
0
    }
420
421
0
    DECLARE_ALIGNED_32(SwsOpExec, tail) = p->exec_tail;
422
0
    tail.slice_y = y;
423
0
    tail.slice_h = h;
424
425
0
    for (int i = 0; i < p->planes_in; i++) {
426
        /* Input offsets are relative to the base pointer */
427
0
        if (!exec.in_offset_x || memcpy_in)
428
0
            exec.in[i] += p->tail_off_in;
429
0
        tail.in[i] += (y_in >> exec.in_sub_y[i]) * tail.in_stride[i];
430
0
    }
431
0
    for (int i = 0; i < p->planes_out; i++) {
432
0
        exec.out[i] += p->tail_off_out;
433
0
        tail.out[i] += y * tail.out_stride[i];
434
0
    }
435
436
0
    for (int i = 0; i < p->planes_in; i++) {
437
0
        if (memcpy_in) {
438
0
            if (!p->offsets_y) { /* already copied by op_pass_setup() */
439
0
                const int lines = h >> exec.in_sub_y[i];
440
0
                copy_lines((uint8_t *) tail.in[i], tail.in_stride[i],
441
0
                           exec.in[i], exec.in_stride[i], lines, p->tail_size_in);
442
0
            }
443
0
        } else {
444
            /* Reuse input pointers directly */
445
0
            const size_t loop_size = tail_blocks * exec.block_size_in[i];
446
0
            tail.in[i]        = exec.in[i];
447
0
            tail.in_stride[i] = exec.in_stride[i];
448
0
            tail.in_bump[i]   = exec.in_stride[i] - loop_size;
449
0
        }
450
0
    }
451
452
0
    for (int i = 0; !memcpy_out && i < p->planes_out; i++) {
453
        /* Reuse output pointers directly */
454
0
        const size_t loop_size = tail_blocks * exec.block_size_out[i];
455
0
        tail.out[i]        = exec.out[i];
456
0
        tail.out_stride[i] = exec.out_stride[i];
457
0
        tail.out_bump[i]   = exec.out_stride[i] - loop_size;
458
0
    }
459
460
    /* Dispatch kernel over tail */
461
0
    av_assert1(tail_blocks > 0);
462
0
    comp->func(&tail, comp->priv, num_blocks - tail_blocks, y, num_blocks, y + h);
463
464
0
    for (int i = 0; memcpy_out && i < p->planes_out; i++) {
465
0
        const int lines = h >> tail.out_sub_y[i];
466
0
        copy_lines(exec.out[i], exec.out_stride[i],
467
0
                   tail.out[i], tail.out_stride[i], lines, p->tail_size_out);
468
0
    }
469
0
}
470
471
/* Updates the plane copy (no-op) map for this operation list */
472
static void op_list_get_plane_copy(const SwsOpList *ops, SwsPass *pass)
473
0
{
474
0
    const SwsOp *write = ff_sws_op_list_output(ops);
475
0
    const SwsOp *read  = ff_sws_op_list_input(ops);
476
0
    if (!write || write->rw.mode != SWS_RW_PLANAR ||
477
0
        !read  || read->rw.mode  != SWS_RW_PLANAR ||
478
0
        read->rw.type != write->rw.type ||
479
0
        read->rw.frac != write->rw.frac)
480
0
        return; /* only regular planes can be directly ref'd */
481
482
0
    const SwsOp *prev = ff_sws_op_prev(write);
483
0
    int *plane_copy = pass->output->plane_copy;
484
0
    for (int i = 0; i < write->rw.elems; i++) {
485
0
        SwsCompFlags flags = prev->outputs[i].flags;
486
0
        if (!(flags & SWS_COMP_COPY))
487
0
            continue;
488
489
0
        const int out_idx = ops->plane_dst[i];
490
0
        switch (prev->outputs[i].dep_in) {
491
0
        case SWS_COMP(0): plane_copy[out_idx] = ops->plane_src[0]; break;
492
0
        case SWS_COMP(1): plane_copy[out_idx] = ops->plane_src[1]; break;
493
0
        case SWS_COMP(2): plane_copy[out_idx] = ops->plane_src[2]; break;
494
0
        case SWS_COMP(3): plane_copy[out_idx] = ops->plane_src[3]; break;
495
0
        }
496
0
    }
497
0
}
498
499
static int rw_data_planes(const SwsOp *op)
500
0
{
501
    /* Exclude the palette plane from the plane count, since it does not need
502
     * to be directly processed/adjusted by the dispatch layer */
503
0
    return op->rw.mode == SWS_RW_PALETTE ? 1 : ff_sws_rw_op_planes(op);
504
0
}
505
506
static int rw_pixel_bits(const SwsOp *op)
507
0
{
508
0
    if (op->rw.mode == SWS_RW_PALETTE)
509
0
        return 8; /* index size */
510
511
0
    int elems = 0;
512
0
    switch (op->rw.mode) {
513
0
    case SWS_RW_PLANAR: elems = 1; break;
514
0
    case SWS_RW_PACKED: elems = op->rw.elems; break;
515
0
    }
516
517
0
    const int size  = ff_sws_pixel_type_size(op->rw.type);
518
0
    const int bits  = 8 >> op->rw.frac;
519
0
    av_assert1(bits >= 1);
520
0
    return elems * size * bits;
521
0
}
522
523
static void align_pass(SwsPass *pass, int block_size, const int *over_rw,
524
                       int pixel_bits)
525
0
{
526
0
    if (!pass || pixel_bits <= 0)
527
0
        return;
528
529
    /* Add at least as many pixels as needed to cover the padding requirement */
530
0
    int pad_max = 0;
531
0
    for (int i = 0; i < 4; i++) {
532
0
        const int pad = (over_rw[i] * 8 + pixel_bits - 1) / pixel_bits;
533
0
        pad_max = FFMAX(pad_max, pad);
534
0
    }
535
536
0
    SwsPassBuffer *buf = pass->output;
537
0
    buf->width_align = FFMAX(buf->width_align, block_size);
538
0
    buf->width_pad = FFMAX(buf->width_pad, pad_max);
539
0
}
540
541
/* Unchanging part of parameter list */
542
typedef struct CompileArgs {
543
    const SwsOpBackend *backend;
544
    SwsGraph *graph;
545
    int flags;
546
} CompileArgs;
547
548
static int compile_single(const CompileArgs *args, const SwsOpList *ops,
549
                          SwsPass *link, SwsPass *input, SwsPass **output)
550
0
{
551
0
    SwsGraph *graph = args->graph;
552
0
    SwsContext *ctx = graph->ctx;
553
0
    SwsOpPass *p = av_mallocz(sizeof(*p));
554
0
    if (!p)
555
0
        return AVERROR(ENOMEM);
556
557
0
    int ret = ff_sws_ops_compile(ctx, args->backend, ops, &p->comp);
558
0
    if (ret < 0)
559
0
        goto fail;
560
0
    else if (args->flags & SWS_OP_FLAG_DRY_RUN)
561
0
        goto fail; /* nothing to do, just return */
562
563
0
    const SwsCompiledOp *comp = &p->comp;
564
0
    const SwsFormat *src = &ops->src;
565
0
    const SwsFormat *dst = &ops->dst;
566
0
    av_assert0(!link || link->format == dst->format);
567
0
    if (p->comp.opaque) {
568
0
        SwsCompiledOp c = *comp;
569
0
        av_free(p);
570
0
        ret = ff_sws_graph_add_pass(graph, dst->format, dst->width, dst->height,
571
0
                                    input, 0, c.slice_align, c.func_opaque,
572
0
                                    NULL, c.priv, c.free, output);
573
0
        if (ret >= 0) {
574
0
            (*output)->backend = c.backend->flags;
575
0
            op_list_get_plane_copy(ops, *output);
576
0
            ff_sws_pass_link_output(*output, link);
577
0
        }
578
0
        return ret;
579
0
    }
580
581
0
    const AVPixFmtDescriptor *indesc  = av_pix_fmt_desc_get(src->format);
582
0
    const AVPixFmtDescriptor *outdesc = av_pix_fmt_desc_get(dst->format);
583
0
    const SwsOp *write = ff_sws_op_list_output(ops);
584
0
    p->planes_out     = rw_data_planes(write);
585
0
    p->pixel_bits_out = rw_pixel_bits(write);
586
0
    p->palette_idx    = -1;
587
0
    p->num_blocks     = (dst->width + comp->block_size - 1) / comp->block_size;
588
0
    if (p->num_blocks * comp->block_size < (unsigned) dst->width)
589
0
        return AVERROR(ERANGE);
590
591
0
    SwsOpExec *exec = &p->exec_base;
592
0
    *exec = (SwsOpExec) {
593
0
        .width  = dst->width,
594
0
        .height = dst->height,
595
0
    };
596
597
0
    const SwsOp *read = ff_sws_op_list_input(ops);
598
0
    if (read) {
599
0
        p->planes_in     = rw_data_planes(read);
600
0
        p->pixel_bits_in = rw_pixel_bits(read);
601
0
        if (read->rw.mode == SWS_RW_PALETTE)
602
0
            p->palette_idx = ops->plane_src[1];
603
0
    }
604
605
0
    const int64_t block_bits_in  = (int64_t) comp->block_size * p->pixel_bits_in;
606
0
    const int64_t block_bits_out = (int64_t) comp->block_size * p->pixel_bits_out;
607
0
    if (block_bits_in & 0x7 || block_bits_out & 0x7) {
608
0
        av_log(ctx, AV_LOG_ERROR, "Block size must be byte-aligned.\n");
609
0
        ret = AVERROR(EINVAL);
610
0
        goto fail;
611
0
    }
612
613
0
    for (int i = 0; i < 4; i++)
614
0
        p->idx_in[i] = p->idx_out[i] = -1;
615
616
0
    for (int i = 0; i < p->planes_in; i++) {
617
0
        const int idx = ops->plane_src[i];
618
0
        const int chroma = idx == 1 || idx == 2;
619
0
        const int sub_x = chroma ? indesc->log2_chroma_w : 0;
620
0
        const int sub_y = chroma ? indesc->log2_chroma_h : 0;
621
0
        exec->in_sub_x[i] = sub_x;
622
0
        exec->in_sub_y[i] = sub_y;
623
0
        exec->block_size_in[i] = block_bits_in >> 3;
624
0
        p->idx_in[i] = idx;
625
0
    }
626
627
0
    for (int i = 0; i < p->planes_out; i++) {
628
0
        const int idx = ops->plane_dst[i];
629
0
        const int chroma = idx == 1 || idx == 2;
630
0
        const int sub_x = chroma ? outdesc->log2_chroma_w : 0;
631
0
        const int sub_y = chroma ? outdesc->log2_chroma_h : 0;
632
0
        exec->out_sub_x[i] = sub_x;
633
0
        exec->out_sub_y[i] = sub_y;
634
0
        exec->block_size_out[i] = block_bits_out >> 3;
635
0
        p->idx_out[i] = idx;
636
0
    }
637
638
0
    const SwsFilterWeights *filter = read ? read->rw.filter.kernel : NULL;
639
0
    if (read && read->rw.filter.op == SWS_OP_FILTER_V) {
640
0
        p->offsets_y = av_refstruct_ref(filter->offsets);
641
0
        p->filter_size_v = filter->filter_size;
642
643
        /* Compute relative pointer bumps for each output line */
644
0
        int32_t *bump = av_malloc_array(filter->dst_size, sizeof(*bump));
645
0
        if (!bump) {
646
0
            ret = AVERROR(ENOMEM);
647
0
            goto fail;
648
0
        }
649
650
0
        int line = filter->offsets[0];
651
0
        for (int y = 0; y < filter->dst_size - 1; y++) {
652
0
            int next = filter->offsets[y + 1];
653
0
            bump[y] = next - line - 1;
654
0
            line = next;
655
0
        }
656
0
        bump[filter->dst_size - 1] = 0;
657
0
        exec->in_bump_y = bump;
658
0
    } else if (read && read->rw.filter.op == SWS_OP_FILTER_H) {
659
        /* Compute pixel offset map for each output line */
660
0
        const int pixels = FFALIGN(filter->dst_size, comp->block_size);
661
0
        int32_t *offset = av_malloc_array(pixels, sizeof(*offset));
662
0
        if (!offset) {
663
0
            ret = AVERROR(ENOMEM);
664
0
            goto fail;
665
0
        }
666
0
        exec->in_offset_x = offset;
667
668
0
        for (int x = 0; x < filter->dst_size; x++) {
669
            /* Sanity check; if the tap would land on a half-pixel, we cannot
670
             * reasonably expect the implementation to know about this. Just
671
             * error out in such (theoretical) cases. */
672
0
            int64_t bits = (int64_t) filter->offsets[x] * p->pixel_bits_in;
673
0
            if ((bits & 0x7) || (bits >> 3) > INT32_MAX) {
674
0
                ret = AVERROR(EINVAL);
675
0
                goto fail;
676
0
            }
677
0
            offset[x] = bits >> 3;
678
0
        }
679
0
        for (int x = filter->dst_size; x < pixels; x++)
680
0
            offset[x] = offset[filter->dst_size - 1];
681
0
        for (int i = 0; i < 4; i++)
682
0
            exec->block_size_in[i] = 0; /* ptr does not advance */
683
0
        p->filter_size_h = filter->filter_size;
684
0
    }
685
686
0
    ret = ff_sws_graph_add_pass(graph, dst->format, dst->width, dst->height,
687
0
                                input, 0, comp->slice_align, op_pass_run,
688
0
                                op_pass_setup, p, op_pass_free, output);
689
0
    if (ret < 0)
690
0
        return ret;
691
692
0
    (*output)->backend = comp->backend->flags;
693
0
    op_list_get_plane_copy(ops, *output);
694
0
    ff_sws_pass_link_output(*output, link);
695
0
    align_pass(*output, comp->block_size, comp->over_write, p->pixel_bits_out);
696
0
    if (read)
697
0
        align_pass(input, comp->block_size, comp->over_read,  p->pixel_bits_in);
698
0
    return 0;
699
700
0
fail:
701
0
    op_pass_free(p);
702
0
    return ret;
703
0
}
704
705
/* Return a mask of all planes matching any flag in `flags` */
706
static SwsCompMask plane_mask_flags(const SwsOp *op, SwsCompFlags flags)
707
0
{
708
0
    SwsCompMask planes = 0;
709
0
    for (int c = 0; c < 4; c++) {
710
0
        if (op->outputs[c].flags & flags)
711
0
            planes |= SWS_COMP(c);
712
0
    }
713
714
0
    return planes;
715
0
}
716
717
/* Takes over ownership of *pops, even on failure */
718
static int compile_subpass(const CompileArgs *args, SwsOpList **pops,
719
                           SwsPass *link, SwsPass *input, SwsPass **output)
720
0
{
721
0
    int ret;
722
0
    SwsContext *ctx = args->graph->ctx;
723
0
    SwsOpList *ops  = *pops;
724
0
    SwsOpList *rest = NULL;
725
0
    SwsPass *tmp = NULL;
726
0
    *pops = NULL;
727
728
0
    if (args->flags & SWS_OP_FLAG_SPLIT_MEMCPY) {
729
        /* Split off copied and constant planes into a separate subpass,
730
         * since these are likely to be handled by the memcpy backend */
731
0
        av_assert0(ops->num_ops >= 2);
732
0
        const SwsOp *write = ff_sws_op_list_output(ops);
733
0
        const SwsOp *prev = ff_sws_op_prev(write);
734
0
        SwsCompMask planes = plane_mask_flags(prev, SWS_COMP_COPY | SWS_COMP_CONST);
735
0
        RET(ff_sws_op_list_split_planes(ops, &rest, planes));
736
0
        if (rest) {
737
            /* Parallel split: share input and link all outputs together */
738
0
            av_log(ctx, AV_LOG_DEBUG, "Splitting const/memcpy planes: %s\n",
739
0
                   ff_sws_comp_mask_str(planes));
740
0
            RET(compile_subpass(args, &ops,  link, input, &tmp));
741
0
            RET(compile_subpass(args, &rest, tmp, input, output));
742
0
            return 0;
743
0
        }
744
0
    }
745
746
0
    ret = compile_single(args, ops, link, input, output);
747
0
    if (ret != AVERROR(ENOTSUP))
748
0
        goto fail; /* either success or a hard error */
749
750
    /* Find any unresolved filter */
751
0
    for (int idx = 1; idx < ops->num_ops - 1; idx++) {
752
0
        const SwsOp *op = &ops->ops[idx];
753
0
        if (op->op == SWS_OP_FILTER_H || op->op == SWS_OP_FILTER_V) {
754
0
            RET(ff_sws_op_list_split_at(ops, &rest, idx));
755
0
            if (ff_sws_op_list_is_noop(ops)) {
756
                /* Prevent infinite recursion by avoiding splitting in a way
757
                 * that does not meaningfully reduce the number of operations
758
                 * performed in the second part. */
759
0
                FFSWAP(SwsOpList *, ops, rest);
760
0
                break;
761
0
            }
762
            /* Serial split: feed first pass into second */
763
0
            RET(compile_subpass(args, &ops,  NULL, input, &tmp));
764
0
            RET(compile_subpass(args, &rest, link, tmp, output));
765
0
            return 0;
766
0
        }
767
0
    }
768
769
    /* If we didn't find any more operations to eliminate, then this ops list
770
     * is simply unsupported by any of the available backends */
771
0
    av_log(ctx, AV_LOG_WARNING, "No backend found for operations:\n");
772
0
    ff_sws_op_list_print(ctx, AV_LOG_WARNING, AV_LOG_TRACE, ops);
773
0
    ret = AVERROR(ENOTSUP);
774
775
0
fail:
776
0
    ff_sws_op_list_free(&ops);
777
0
    ff_sws_op_list_free(&rest);
778
0
    return ret;
779
0
}
780
781
int ff_sws_compile_pass(SwsGraph *graph, const SwsOpBackend *backend,
782
                        SwsOpList **pops, int flags, SwsPass *input,
783
                        SwsPass **output)
784
0
{
785
0
    const int passes_orig = graph->num_passes;
786
0
    SwsContext *ctx = graph->ctx;
787
0
    SwsOpList *ops = *pops;
788
0
    int ret = 0;
789
790
0
    const SwsOp *write = ff_sws_op_list_output(ops);
791
0
    if (!write) {
792
0
        av_log(ctx, AV_LOG_ERROR, "Last operation must be SWS_OP_WRITE.\n");
793
0
        ret = AVERROR(EINVAL);
794
0
        goto out;
795
0
    }
796
797
0
    if (flags & SWS_OP_FLAG_OPTIMIZE) {
798
0
        ret = ff_sws_op_list_optimize(ops);
799
0
        if (ret < 0)
800
0
            goto out;
801
0
        av_log(ctx, AV_LOG_DEBUG, "Operation list after optimizing:\n");
802
0
        ff_sws_op_list_print(ctx, AV_LOG_DEBUG, AV_LOG_TRACE, ops);
803
0
    }
804
805
    /* Check if the whole operation graph is an end-to-end no-op */
806
0
    if (ff_sws_op_list_is_noop(ops)) {
807
0
        if (output)
808
0
            *output = input;
809
0
        goto out;
810
0
    }
811
812
0
    const CompileArgs args = {
813
0
        .backend = backend,
814
0
        .graph   = graph,
815
0
        .flags   = flags,
816
0
    };
817
818
0
    ret = compile_subpass(&args, &ops, NULL, input, output);
819
0
    if (ret < 0)
820
0
        goto out;
821
822
0
    const int num_passes = graph->num_passes - passes_orig;
823
0
    if (num_passes > 1)
824
0
        av_log(ctx, AV_LOG_VERBOSE, "Using %d separate passes.\n", num_passes);
825
826
0
out:
827
0
    if (ret < 0)
828
0
        ff_sws_graph_rollback(graph, passes_orig);
829
0
    ff_sws_op_list_free(&ops);
830
    *pops = NULL;
831
0
    return ret;
832
0
}