/src/llama.cpp/src/models/wavtokenizer-dec.cpp
Line | Count | Source |
1 | | #include "models.h" |
2 | | |
3 | 0 | llm_build_wavtokenizer_dec::llm_build_wavtokenizer_dec(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { |
4 | 0 | ggml_tensor * cur; |
5 | 0 | ggml_tensor * inpL; |
6 | |
|
7 | 0 | inpL = build_inp_embd(model.tok_embd); |
8 | |
|
9 | 0 | cur = ggml_cont(ctx0, ggml_transpose(ctx0, inpL)); |
10 | |
|
11 | 0 | cur = ggml_conv_1d_ph(ctx0, model.conv1d, cur, 1, 1); |
12 | 0 | cur = ggml_add(ctx0, cur, model.conv1d_b); |
13 | | |
14 | | // posnet |
15 | 0 | for (uint32_t il = 0; il < hparams.posnet.n_layer; ++il) { |
16 | 0 | const auto & layer = model.layers[il].posnet; |
17 | |
|
18 | 0 | inpL = cur; |
19 | |
|
20 | 0 | switch (il) { |
21 | 0 | case 0: |
22 | 0 | case 1: |
23 | 0 | case 3: |
24 | 0 | case 4: |
25 | 0 | { |
26 | 0 | cur = build_norm(cur, |
27 | 0 | layer.norm1, |
28 | 0 | layer.norm1_b, |
29 | 0 | LLM_NORM_GROUP, 0); |
30 | |
|
31 | 0 | cur = ggml_mul(ctx0, ggml_sigmoid(ctx0, cur), cur); |
32 | |
|
33 | 0 | cur = ggml_conv_1d_ph(ctx0, layer.conv1, cur, 1, 1); |
34 | 0 | cur = ggml_add(ctx0, cur, layer.conv1_b); |
35 | |
|
36 | 0 | cur = build_norm(cur, |
37 | 0 | layer.norm2, |
38 | 0 | layer.norm2_b, |
39 | 0 | LLM_NORM_GROUP, 0); |
40 | |
|
41 | 0 | cur = ggml_mul(ctx0, ggml_sigmoid(ctx0, cur), cur); |
42 | |
|
43 | 0 | cur = ggml_conv_1d_ph(ctx0, layer.conv2, cur, 1, 1); |
44 | 0 | cur = ggml_add(ctx0, cur, layer.conv2_b); |
45 | |
|
46 | 0 | cur = ggml_add(ctx0, cur, inpL); |
47 | 0 | } break; |
48 | 0 | case 2: |
49 | 0 | { |
50 | 0 | cur = build_norm(cur, |
51 | 0 | layer.attn_norm, |
52 | 0 | layer.attn_norm_b, |
53 | 0 | LLM_NORM_GROUP, 0); |
54 | |
|
55 | 0 | ggml_tensor * q; |
56 | 0 | ggml_tensor * k; |
57 | 0 | ggml_tensor * v; |
58 | |
|
59 | 0 | q = ggml_conv_1d_ph(ctx0, layer.attn_q, cur, 1, 1); |
60 | 0 | k = ggml_conv_1d_ph(ctx0, layer.attn_k, cur, 1, 1); |
61 | 0 | v = ggml_conv_1d_ph(ctx0, layer.attn_v, cur, 1, 1); |
62 | |
|
63 | 0 | q = ggml_add(ctx0, q, layer.attn_q_b); |
64 | 0 | k = ggml_add(ctx0, k, layer.attn_k_b); |
65 | 0 | v = ggml_add(ctx0, v, layer.attn_v_b); |
66 | |
|
67 | 0 | q = ggml_cont(ctx0, ggml_transpose(ctx0, q)); |
68 | 0 | k = ggml_cont(ctx0, ggml_transpose(ctx0, k)); |
69 | |
|
70 | 0 | ggml_tensor * kq = ggml_mul_mat(ctx0, k, q); |
71 | |
|
72 | 0 | kq = ggml_soft_max_ext(ctx0, kq, nullptr, 1.0f/sqrtf(float(hparams.posnet.n_embd)), 0.0f); |
73 | |
|
74 | 0 | cur = ggml_mul_mat(ctx0, kq, v); |
75 | |
|
76 | 0 | cur = ggml_conv_1d_ph(ctx0, layer.attn_o, cur, 1, 1); |
77 | 0 | cur = ggml_add(ctx0, cur, layer.attn_o_b); |
78 | |
|
79 | 0 | cur = ggml_add(ctx0, cur, inpL); |
80 | 0 | } break; |
81 | 0 | case 5: |
82 | 0 | { |
83 | 0 | cur = build_norm(cur, |
84 | 0 | layer.norm, |
85 | 0 | layer.norm_b, |
86 | 0 | LLM_NORM_GROUP, 0); |
87 | 0 | } break; |
88 | 0 | default: GGML_ABORT("unknown posnet layer"); |
89 | 0 | }; |
90 | 0 | } |
91 | 0 | cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); |
92 | |
|
93 | 0 | cur = build_norm(cur, |
94 | 0 | model.tok_norm, |
95 | 0 | model.tok_norm_b, |
96 | 0 | LLM_NORM, -1); |
97 | |
|
98 | 0 | cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); |
99 | |
|
100 | 0 | inpL = cur; |
101 | | |
102 | | // convnext |
103 | 0 | for (uint32_t il = 0; il < hparams.convnext.n_layer; ++il) { |
104 | 0 | const auto & layer = model.layers[il].convnext; |
105 | |
|
106 | 0 | cur = inpL; |
107 | |
|
108 | 0 | cur = ggml_conv_1d_dw_ph(ctx0, layer.dw, cur, 1, 1); |
109 | 0 | cur = ggml_add(ctx0, cur, layer.dw_b); |
110 | |
|
111 | 0 | cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); |
112 | |
|
113 | 0 | cur = build_norm(cur, |
114 | 0 | layer.norm, |
115 | 0 | layer.norm_b, |
116 | 0 | LLM_NORM, -1); |
117 | |
|
118 | 0 | cur = build_ffn(cur, |
119 | 0 | layer.pw1, layer.pw1_b, NULL, |
120 | 0 | NULL, NULL, NULL, |
121 | 0 | layer.pw2, layer.pw2_b, NULL, |
122 | 0 | NULL, |
123 | 0 | LLM_FFN_GELU, LLM_FFN_SEQ, il); |
124 | |
|
125 | 0 | cur = ggml_mul(ctx0, cur, layer.gamma); |
126 | |
|
127 | 0 | cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); |
128 | |
|
129 | 0 | inpL = ggml_add(ctx0, cur, inpL); |
130 | 0 | } |
131 | 0 | cur = inpL; |
132 | |
|
133 | 0 | cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); |
134 | |
|
135 | 0 | cur = build_norm(cur, |
136 | 0 | model.output_norm, |
137 | 0 | model.output_norm_b, |
138 | 0 | LLM_NORM, -1); |
139 | | |
140 | | // lm_head |
141 | 0 | cur = build_lora_mm(model.output, cur); |
142 | |
|
143 | 0 | cur = ggml_add(ctx0, cur, model.output_b); |
144 | |
|
145 | 0 | cb(cur, "result_embd", -1); |
146 | 0 | res->t_embd = cur; |
147 | |
|
148 | 0 | ggml_build_forward_expand(gf, cur); |
149 | 0 | } |