/src/llama.cpp/src/models/wavtokenizer-dec.cpp

Source
#include "models.h"

void llama_model_wavtokenizer_dec::load_arch_hparams(llama_model_loader & ml) {
    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS,    hparams.f_norm_eps);
    ml.get_key(LLM_KV_ATTENTION_GROUPNORM_EPS,    hparams.f_norm_group_eps);
    ml.get_key(LLM_KV_ATTENTION_GROUPNORM_GROUPS, hparams.n_norm_groups);
}

void llama_model_wavtokenizer_dec::load_arch_tensors(llama_model_loader &) {
    LLAMA_LOAD_LOCALS;

    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {hparams.n_embd, n_vocab}, 0);

    conv1d   = create_tensor(tn(LLM_TENSOR_CONV1D, "weight", 0), {7, hparams.n_embd, hparams.posnet.n_embd}, 0);
    conv1d_b = create_tensor(tn(LLM_TENSOR_CONV1D, "bias",   0), {1, hparams.posnet.n_embd}, 0);

    // posnet
    {
        const int64_t n_embd = hparams.posnet.n_embd;

        for (uint32_t i = 0; i < hparams.posnet.n_layer; ++i) {
            auto & layer = layers[i].posnet;

            // posnet:
            //
            //  - resnet
            //  - resnet
            //  - attn
            //  - resnet
            //  - resnet
            //  - norm
            //
            switch (i) {
                case 0:
                case 1:
                case 3:
                case 4:
                    {
                        layer.norm1   = create_tensor(tn(LLM_TENSOR_POS_NET_NORM1, "weight", i), {1, n_embd}, 0);
                        layer.norm1_b = create_tensor(tn(LLM_TENSOR_POS_NET_NORM1, "bias",   i), {1, n_embd}, 0);

                        layer.conv1   = create_tensor(tn(LLM_TENSOR_POS_NET_CONV1, "weight", i), {3, n_embd, n_embd}, 0);
                        layer.conv1_b = create_tensor(tn(LLM_TENSOR_POS_NET_CONV1, "bias",   i), {1, n_embd}, 0);

                        layer.norm2   = create_tensor(tn(LLM_TENSOR_POS_NET_NORM2, "weight", i), {1, n_embd}, 0);
                        layer.norm2_b = create_tensor(tn(LLM_TENSOR_POS_NET_NORM2, "bias",   i), {1, n_embd}, 0);

                        layer.conv2   = create_tensor(tn(LLM_TENSOR_POS_NET_CONV2, "weight", i), {3, n_embd, n_embd}, 0);
                        layer.conv2_b = create_tensor(tn(LLM_TENSOR_POS_NET_CONV2, "bias",   i), {1, n_embd}, 0);
                    } break;
                case 2:
                    {
                        layer.attn_norm   = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_NORM, "weight", i), {1, n_embd}, 0);
                        layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_NORM, "bias",   i), {1, n_embd}, 0);

                        layer.attn_q      = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_Q,    "weight", i), {1, n_embd, n_embd}, 0);
                        layer.attn_q_b    = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_Q,    "bias",   i), {1, n_embd}, 0);

                        layer.attn_k      = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_K,    "weight", i), {1, n_embd, n_embd}, 0);
                        layer.attn_k_b    = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_K,    "bias",   i), {1, n_embd}, 0);

                        layer.attn_v      = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_V,    "weight", i), {1, n_embd, n_embd}, 0);
                        layer.attn_v_b    = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_V,    "bias",   i), {1, n_embd}, 0);

                        layer.attn_o      = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_OUT,  "weight", i), {1, n_embd, n_embd}, 0);
                        layer.attn_o_b    = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_OUT,  "bias",   i), {1, n_embd}, 0);
                    } break;
                case 5:
                    {
                        layer.norm   = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_NORM, "weight", i), {1, n_embd}, 0);
                        layer.norm_b = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_NORM, "bias",   i), {1, n_embd}, 0);
                    } break;
                default: GGML_ABORT("unknown posnet layer");
            };
        }
    }

    GGML_ASSERT(hparams.posnet.n_embd == hparams.convnext.n_embd);

    tok_norm   = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {hparams.posnet.n_embd}, 0);
    tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias",   0), {hparams.posnet.n_embd}, 0);

    // convnext
    {
        const int64_t n_embd = hparams.convnext.n_embd;

        for (uint32_t i = 0; i < hparams.convnext.n_layer; ++i) {
            auto & layer = layers[i].convnext;

            layer.dw     = create_tensor(tn(LLM_TENSOR_CONVNEXT_DW,    "weight", i), {7, 1, n_embd}, 0);
            layer.dw_b   = create_tensor(tn(LLM_TENSOR_CONVNEXT_DW,    "bias",   i), {1, n_embd}, 0);

            layer.norm   = create_tensor(tn(LLM_TENSOR_CONVNEXT_NORM,  "weight", i), {n_embd}, 0);
            layer.norm_b = create_tensor(tn(LLM_TENSOR_CONVNEXT_NORM,  "bias",   i), {n_embd}, 0);

            layer.pw1    = create_tensor(tn(LLM_TENSOR_CONVNEXT_PW1,   "weight", i), {n_embd, n_ff}, 0);
            layer.pw1_b  = create_tensor(tn(LLM_TENSOR_CONVNEXT_PW1,   "bias",   i), {n_ff}, 0);

            layer.pw2    = create_tensor(tn(LLM_TENSOR_CONVNEXT_PW2,   "weight", i), {n_ff, n_embd}, 0);
            layer.pw2_b  = create_tensor(tn(LLM_TENSOR_CONVNEXT_PW2,   "bias",   i), {n_embd}, 0);

            layer.gamma  = create_tensor(tn(LLM_TENSOR_CONVNEXT_GAMMA, "weight", i), {n_embd}, 0);
        }

        // output
        output_norm   = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
        output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "bias"),   {n_embd}, 0);
    }

    output   = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {hparams.convnext.n_embd, hparams.n_embd_out()}, 0);
    output_b = create_tensor(tn(LLM_TENSOR_OUTPUT, "bias"),   {hparams.n_embd_out()}, 0);
}

std::unique_ptr<llm_graph_context> llama_model_wavtokenizer_dec::build_arch_graph(const llm_graph_params & params) const {
    return std::make_unique<graph>(*this, params);
}

llama_model_wavtokenizer_dec::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
    ggml_tensor * cur;
    ggml_tensor * inpL;

    inpL = build_inp_embd(model.tok_embd);

    cur = ggml_cont(ctx0, ggml_transpose(ctx0, inpL));

    cur = ggml_conv_1d_ph(ctx0, model.conv1d, cur, 1, 1);
    cur = ggml_add(ctx0, cur, model.conv1d_b);

    // posnet
    for (uint32_t il = 0; il < hparams.posnet.n_layer; ++il) {
        const auto & layer = model.layers[il].posnet;

        inpL = cur;

        switch (il) {
            case 0:
            case 1:
            case 3:
            case 4:
                {
                    cur = build_norm(cur,
                            layer.norm1,
                            layer.norm1_b,
                            LLM_NORM_GROUP, 0);

                    cur = ggml_mul(ctx0, ggml_sigmoid(ctx0, cur), cur);

                    cur = ggml_conv_1d_ph(ctx0, layer.conv1, cur, 1, 1);
                    cur = ggml_add(ctx0, cur, layer.conv1_b);

                    cur = build_norm(cur,
                            layer.norm2,
                            layer.norm2_b,
                            LLM_NORM_GROUP, 0);

                    cur = ggml_mul(ctx0, ggml_sigmoid(ctx0, cur), cur);

                    cur = ggml_conv_1d_ph(ctx0, layer.conv2, cur, 1, 1);
                    cur = ggml_add(ctx0, cur, layer.conv2_b);

                    cur = ggml_add(ctx0, cur, inpL);
                } break;
            case 2:
                {
                    cur = build_norm(cur,
                            layer.attn_norm,
                            layer.attn_norm_b,
                            LLM_NORM_GROUP, 0);

                    ggml_tensor * q;
                    ggml_tensor * k;
                    ggml_tensor * v;

                    q = ggml_conv_1d_ph(ctx0, layer.attn_q, cur, 1, 1);
                    k = ggml_conv_1d_ph(ctx0, layer.attn_k, cur, 1, 1);
                    v = ggml_conv_1d_ph(ctx0, layer.attn_v, cur, 1, 1);

                    q = ggml_add(ctx0, q, layer.attn_q_b);
                    k = ggml_add(ctx0, k, layer.attn_k_b);
                    v = ggml_add(ctx0, v, layer.attn_v_b);

                    q = ggml_cont(ctx0, ggml_transpose(ctx0, q));
                    k = ggml_cont(ctx0, ggml_transpose(ctx0, k));

                    ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);

                    kq = ggml_soft_max_ext(ctx0, kq, nullptr, 1.0f/sqrtf(float(hparams.posnet.n_embd)), 0.0f);

                    cur = ggml_mul_mat(ctx0, kq, v);

                    cur = ggml_conv_1d_ph(ctx0, layer.attn_o, cur, 1, 1);
                    cur = ggml_add(ctx0, cur, layer.attn_o_b);

                    cur = ggml_add(ctx0, cur, inpL);
                } break;
            case 5:
                {
                    cur = build_norm(cur,
                            layer.norm,
                            layer.norm_b,
                            LLM_NORM_GROUP, 0);
                } break;
            default: GGML_ABORT("unknown posnet layer");
        };
    }
    cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));

    cur = build_norm(cur,
            model.tok_norm,
            model.tok_norm_b,
            LLM_NORM, 0);

    cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));

    inpL = cur;

    // convnext
    for (uint32_t il = 0; il < hparams.convnext.n_layer; ++il) {
        const auto & layer = model.layers[il].convnext;

        cur = inpL;

        cur = ggml_conv_1d_dw_ph(ctx0, layer.dw, cur, 1, 1);
        cur = ggml_add(ctx0, cur, layer.dw_b);

        cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));

        cur = build_norm(cur,
                layer.norm,
                layer.norm_b,
                LLM_NORM, -1);

        cur = build_ffn(cur,
                layer.pw1, layer.pw1_b, NULL,
                NULL,      NULL,        NULL,
                layer.pw2, layer.pw2_b, NULL,
                NULL,
                LLM_FFN_GELU, LLM_FFN_SEQ, il);

        cur = ggml_mul(ctx0, cur, layer.gamma);

        cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));

        inpL = ggml_add(ctx0, cur, inpL);
    }
    cur = inpL;

    cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));

    cur = build_norm(cur,
            model.output_norm,
            model.output_norm_b,
            LLM_NORM, -1);

    // lm_head
    cur = build_lora_mm(model.output, cur, model.output_s);

    cur = ggml_add(ctx0, cur, model.output_b);

    cb(cur, "result_embd", -1);
    res->t_embd = cur;

    ggml_build_forward_expand(gf, cur);
}

Coverage Report

Created: 2026-07-16 06:35

Line	Count	Source
1		#include "models.h"
2
3	0	void llama_model_wavtokenizer_dec::load_arch_hparams(llama_model_loader & ml) {
4	0	ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
5	0	ml.get_key(LLM_KV_ATTENTION_GROUPNORM_EPS, hparams.f_norm_group_eps);
6	0	ml.get_key(LLM_KV_ATTENTION_GROUPNORM_GROUPS, hparams.n_norm_groups);
7	0	}
8
9	0	void llama_model_wavtokenizer_dec::load_arch_tensors(llama_model_loader &) {
10	0	LLAMA_LOAD_LOCALS;
11
12	0	tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {hparams.n_embd, n_vocab}, 0);
13
14	0	conv1d = create_tensor(tn(LLM_TENSOR_CONV1D, "weight", 0), {7, hparams.n_embd, hparams.posnet.n_embd}, 0);
15	0	conv1d_b = create_tensor(tn(LLM_TENSOR_CONV1D, "bias", 0), {1, hparams.posnet.n_embd}, 0);
16
17		// posnet
18	0	{
19	0	const int64_t n_embd = hparams.posnet.n_embd;
20
21	0	for (uint32_t i = 0; i < hparams.posnet.n_layer; ++i) {
22	0	auto & layer = layers[i].posnet;
23
24		// posnet:
25		//
26		// - resnet
27		// - resnet
28		// - attn
29		// - resnet
30		// - resnet
31		// - norm
32		//
33	0	switch (i) {
34	0	case 0:
35	0	case 1:
36	0	case 3:
37	0	case 4:
38	0	{
39	0	layer.norm1 = create_tensor(tn(LLM_TENSOR_POS_NET_NORM1, "weight", i), {1, n_embd}, 0);
40	0	layer.norm1_b = create_tensor(tn(LLM_TENSOR_POS_NET_NORM1, "bias", i), {1, n_embd}, 0);
41
42	0	layer.conv1 = create_tensor(tn(LLM_TENSOR_POS_NET_CONV1, "weight", i), {3, n_embd, n_embd}, 0);
43	0	layer.conv1_b = create_tensor(tn(LLM_TENSOR_POS_NET_CONV1, "bias", i), {1, n_embd}, 0);
44
45	0	layer.norm2 = create_tensor(tn(LLM_TENSOR_POS_NET_NORM2, "weight", i), {1, n_embd}, 0);
46	0	layer.norm2_b = create_tensor(tn(LLM_TENSOR_POS_NET_NORM2, "bias", i), {1, n_embd}, 0);
47
48	0	layer.conv2 = create_tensor(tn(LLM_TENSOR_POS_NET_CONV2, "weight", i), {3, n_embd, n_embd}, 0);
49	0	layer.conv2_b = create_tensor(tn(LLM_TENSOR_POS_NET_CONV2, "bias", i), {1, n_embd}, 0);
50	0	} break;
51	0	case 2:
52	0	{
53	0	layer.attn_norm = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_NORM, "weight", i), {1, n_embd}, 0);
54	0	layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_NORM, "bias", i), {1, n_embd}, 0);
55
56	0	layer.attn_q = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_Q, "weight", i), {1, n_embd, n_embd}, 0);
57	0	layer.attn_q_b = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_Q, "bias", i), {1, n_embd}, 0);
58
59	0	layer.attn_k = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_K, "weight", i), {1, n_embd, n_embd}, 0);
60	0	layer.attn_k_b = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_K, "bias", i), {1, n_embd}, 0);
61
62	0	layer.attn_v = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_V, "weight", i), {1, n_embd, n_embd}, 0);
63	0	layer.attn_v_b = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_V, "bias", i), {1, n_embd}, 0);
64
65	0	layer.attn_o = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_OUT, "weight", i), {1, n_embd, n_embd}, 0);
66	0	layer.attn_o_b = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_OUT, "bias", i), {1, n_embd}, 0);
67	0	} break;
68	0	case 5:
69	0	{
70	0	layer.norm = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_NORM, "weight", i), {1, n_embd}, 0);
71	0	layer.norm_b = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_NORM, "bias", i), {1, n_embd}, 0);
72	0	} break;
73	0	default: GGML_ABORT("unknown posnet layer");
74	0	};
75	0	}
76	0	}
77
78	0	GGML_ASSERT(hparams.posnet.n_embd == hparams.convnext.n_embd);
79
80	0	tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {hparams.posnet.n_embd}, 0);
81	0	tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias", 0), {hparams.posnet.n_embd}, 0);
82
83		// convnext
84	0	{
85	0	const int64_t n_embd = hparams.convnext.n_embd;
86
87	0	for (uint32_t i = 0; i < hparams.convnext.n_layer; ++i) {
88	0	auto & layer = layers[i].convnext;
89
90	0	layer.dw = create_tensor(tn(LLM_TENSOR_CONVNEXT_DW, "weight", i), {7, 1, n_embd}, 0);
91	0	layer.dw_b = create_tensor(tn(LLM_TENSOR_CONVNEXT_DW, "bias", i), {1, n_embd}, 0);
92
93	0	layer.norm = create_tensor(tn(LLM_TENSOR_CONVNEXT_NORM, "weight", i), {n_embd}, 0);
94	0	layer.norm_b = create_tensor(tn(LLM_TENSOR_CONVNEXT_NORM, "bias", i), {n_embd}, 0);
95
96	0	layer.pw1 = create_tensor(tn(LLM_TENSOR_CONVNEXT_PW1, "weight", i), {n_embd, n_ff}, 0);
97	0	layer.pw1_b = create_tensor(tn(LLM_TENSOR_CONVNEXT_PW1, "bias", i), {n_ff}, 0);
98
99	0	layer.pw2 = create_tensor(tn(LLM_TENSOR_CONVNEXT_PW2, "weight", i), {n_ff, n_embd}, 0);
100	0	layer.pw2_b = create_tensor(tn(LLM_TENSOR_CONVNEXT_PW2, "bias", i), {n_embd}, 0);
101
102	0	layer.gamma = create_tensor(tn(LLM_TENSOR_CONVNEXT_GAMMA, "weight", i), {n_embd}, 0);
103	0	}
104
105		// output
106	0	output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
107	0	output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, 0);
108	0	}
109
110	0	output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {hparams.convnext.n_embd, hparams.n_embd_out()}, 0);
111	0	output_b = create_tensor(tn(LLM_TENSOR_OUTPUT, "bias"), {hparams.n_embd_out()}, 0);
112	0	}
113
114	0	std::unique_ptr<llm_graph_context> llama_model_wavtokenizer_dec::build_arch_graph(const llm_graph_params & params) const {
115	0	return std::make_unique<graph>(*this, params);
116	0	}
117
118	0	llama_model_wavtokenizer_dec::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
119	0	ggml_tensor * cur;
120	0	ggml_tensor * inpL;
121
122	0	inpL = build_inp_embd(model.tok_embd);
123
124	0	cur = ggml_cont(ctx0, ggml_transpose(ctx0, inpL));
125
126	0	cur = ggml_conv_1d_ph(ctx0, model.conv1d, cur, 1, 1);
127	0	cur = ggml_add(ctx0, cur, model.conv1d_b);
128
129		// posnet
130	0	for (uint32_t il = 0; il < hparams.posnet.n_layer; ++il) {
131	0	const auto & layer = model.layers[il].posnet;
132
133	0	inpL = cur;
134
135	0	switch (il) {
136	0	case 0:
137	0	case 1:
138	0	case 3:
139	0	case 4:
140	0	{
141	0	cur = build_norm(cur,
142	0	layer.norm1,
143	0	layer.norm1_b,
144	0	LLM_NORM_GROUP, 0);
145
146	0	cur = ggml_mul(ctx0, ggml_sigmoid(ctx0, cur), cur);
147
148	0	cur = ggml_conv_1d_ph(ctx0, layer.conv1, cur, 1, 1);
149	0	cur = ggml_add(ctx0, cur, layer.conv1_b);
150
151	0	cur = build_norm(cur,
152	0	layer.norm2,
153	0	layer.norm2_b,
154	0	LLM_NORM_GROUP, 0);
155
156	0	cur = ggml_mul(ctx0, ggml_sigmoid(ctx0, cur), cur);
157
158	0	cur = ggml_conv_1d_ph(ctx0, layer.conv2, cur, 1, 1);
159	0	cur = ggml_add(ctx0, cur, layer.conv2_b);
160
161	0	cur = ggml_add(ctx0, cur, inpL);
162	0	} break;
163	0	case 2:
164	0	{
165	0	cur = build_norm(cur,
166	0	layer.attn_norm,
167	0	layer.attn_norm_b,
168	0	LLM_NORM_GROUP, 0);
169
170	0	ggml_tensor * q;
171	0	ggml_tensor * k;
172	0	ggml_tensor * v;
173
174	0	q = ggml_conv_1d_ph(ctx0, layer.attn_q, cur, 1, 1);
175	0	k = ggml_conv_1d_ph(ctx0, layer.attn_k, cur, 1, 1);
176	0	v = ggml_conv_1d_ph(ctx0, layer.attn_v, cur, 1, 1);
177
178	0	q = ggml_add(ctx0, q, layer.attn_q_b);
179	0	k = ggml_add(ctx0, k, layer.attn_k_b);
180	0	v = ggml_add(ctx0, v, layer.attn_v_b);
181
182	0	q = ggml_cont(ctx0, ggml_transpose(ctx0, q));
183	0	k = ggml_cont(ctx0, ggml_transpose(ctx0, k));
184
185	0	ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);
186
187	0	kq = ggml_soft_max_ext(ctx0, kq, nullptr, 1.0f/sqrtf(float(hparams.posnet.n_embd)), 0.0f);
188
189	0	cur = ggml_mul_mat(ctx0, kq, v);
190
191	0	cur = ggml_conv_1d_ph(ctx0, layer.attn_o, cur, 1, 1);
192	0	cur = ggml_add(ctx0, cur, layer.attn_o_b);
193
194	0	cur = ggml_add(ctx0, cur, inpL);
195	0	} break;
196	0	case 5:
197	0	{
198	0	cur = build_norm(cur,
199	0	layer.norm,
200	0	layer.norm_b,
201	0	LLM_NORM_GROUP, 0);
202	0	} break;
203	0	default: GGML_ABORT("unknown posnet layer");
204	0	};
205	0	}
206	0	cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
207
208	0	cur = build_norm(cur,
209	0	model.tok_norm,
210	0	model.tok_norm_b,
211	0	LLM_NORM, 0);
212
213	0	cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
214
215	0	inpL = cur;
216
217		// convnext
218	0	for (uint32_t il = 0; il < hparams.convnext.n_layer; ++il) {
219	0	const auto & layer = model.layers[il].convnext;
220
221	0	cur = inpL;
222
223	0	cur = ggml_conv_1d_dw_ph(ctx0, layer.dw, cur, 1, 1);
224	0	cur = ggml_add(ctx0, cur, layer.dw_b);
225
226	0	cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
227
228	0	cur = build_norm(cur,
229	0	layer.norm,
230	0	layer.norm_b,
231	0	LLM_NORM, -1);
232
233	0	cur = build_ffn(cur,
234	0	layer.pw1, layer.pw1_b, NULL,
235	0	NULL, NULL, NULL,
236	0	layer.pw2, layer.pw2_b, NULL,
237	0	NULL,
238	0	LLM_FFN_GELU, LLM_FFN_SEQ, il);
239
240	0	cur = ggml_mul(ctx0, cur, layer.gamma);
241
242	0	cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
243
244	0	inpL = ggml_add(ctx0, cur, inpL);
245	0	}
246	0	cur = inpL;
247
248	0	cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
249
250	0	cur = build_norm(cur,
251	0	model.output_norm,
252	0	model.output_norm_b,
253	0	LLM_NORM, -1);
254
255		// lm_head
256	0	cur = build_lora_mm(model.output, cur, model.output_s);
257
258	0	cur = ggml_add(ctx0, cur, model.output_b);
259
260	0	cb(cur, "result_embd", -1);
261	0	res->t_embd = cur;
262
263	0	ggml_build_forward_expand(gf, cur);
264	0	}