/src/tesseract/src/lstm/functions.h
Line | Count | Source (jump to first uncovered line) |
1 | | /////////////////////////////////////////////////////////////////////// |
2 | | // File: functions.h |
3 | | // Description: Collection of function-objects used by the network layers. |
4 | | // Author: Ray Smith |
5 | | // |
6 | | // (C) Copyright 2014, Google Inc. |
7 | | // Licensed under the Apache License, Version 2.0 (the "License"); |
8 | | // you may not use this file except in compliance with the License. |
9 | | // You may obtain a copy of the License at |
10 | | // http://www.apache.org/licenses/LICENSE-2.0 |
11 | | // Unless required by applicable law or agreed to in writing, software |
12 | | // distributed under the License is distributed on an "AS IS" BASIS, |
13 | | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
14 | | // See the License for the specific language governing permissions and |
15 | | // limitations under the License. |
16 | | /////////////////////////////////////////////////////////////////////// |
17 | | |
18 | | #ifndef TESSERACT_LSTM_FUNCTIONS_H_ |
19 | | #define TESSERACT_LSTM_FUNCTIONS_H_ |
20 | | |
21 | | #include "helpers.h" |
22 | | #include "tesstypes.h" |
23 | | |
24 | | // Setting this to 1 or more causes massive dumps of debug data: weights, |
25 | | // updates, internal calculations etc, and reduces the number of test iterations |
26 | | // to a small number, so outputs can be diffed. |
27 | | #define DEBUG_DETAIL 0 |
28 | | #if DEBUG_DETAIL > 0 |
29 | | # undef _OPENMP // Disable open mp to get the outputs in sync. |
30 | | #endif |
31 | | |
32 | | namespace tesseract { |
33 | | |
34 | | // Size of static tables. |
35 | | constexpr int kTableSize = 4096; |
36 | | // Scale factor for float arg to int index. |
37 | | constexpr TFloat kScaleFactor = 256.0; |
38 | | |
39 | | // Generated lookup tables. |
40 | | extern const TFloat TanhTable[]; |
41 | | extern const TFloat LogisticTable[]; |
42 | | |
43 | | // Non-linearity (sigmoid) functions with cache tables and clipping. |
44 | 24.9G | inline TFloat Tanh(TFloat x) { |
45 | 24.9G | if (x < 0) { |
46 | 9.27G | return -Tanh(-x); |
47 | 9.27G | } |
48 | 15.7G | x *= kScaleFactor; |
49 | 15.7G | auto index = static_cast<unsigned>(x); |
50 | 15.7G | if (index >= (kTableSize - 1)) { |
51 | 3.65M | return 1; |
52 | 3.65M | } |
53 | 15.7G | TFloat tanh_i0 = TanhTable[index]; |
54 | 15.7G | TFloat tanh_i1 = TanhTable[index + 1]; |
55 | | // Linear interpolation. |
56 | 15.7G | return tanh_i0 + (tanh_i1 - tanh_i0) * (x - index); |
57 | 15.7G | } |
58 | | |
59 | 25.1G | inline TFloat Logistic(TFloat x) { |
60 | 25.1G | if (x < 0) { |
61 | 10.4G | return 1 - Logistic(-x); |
62 | 10.4G | } |
63 | 14.7G | x *= kScaleFactor; |
64 | 14.7G | auto index = static_cast<unsigned>(x); |
65 | 14.7G | if (index >= (kTableSize - 1)) { |
66 | 117k | return 1; |
67 | 117k | } |
68 | 14.7G | TFloat l0 = LogisticTable[index]; |
69 | 14.7G | TFloat l1 = LogisticTable[index + 1]; |
70 | | // Linear interpolation. |
71 | 14.7G | return l0 + (l1 - l0) * (x - index); |
72 | 14.7G | } |
73 | | |
74 | | // Non-linearity (sigmoid) functions and their derivatives. |
75 | | struct FFunc { |
76 | 14.7G | inline TFloat operator()(TFloat x) const { |
77 | 14.7G | return Logistic(x); |
78 | 14.7G | } |
79 | | }; |
80 | | struct FPrime { |
81 | 0 | inline TFloat operator()(TFloat y) const { |
82 | 0 | return y * (1 - y); |
83 | 0 | } |
84 | | }; |
85 | | struct ClipFFunc { |
86 | 0 | inline TFloat operator()(TFloat x) const { |
87 | 0 | if (x <= 0) { |
88 | 0 | return 0; |
89 | 0 | } |
90 | 0 | if (x >= 1) { |
91 | 0 | return 1; |
92 | 0 | } |
93 | 0 | return x; |
94 | 0 | } |
95 | | }; |
96 | | struct ClipFPrime { |
97 | 0 | inline TFloat operator()(TFloat y) const { |
98 | 0 | return 0 < y && y < 1 ? 1 : 0; |
99 | 0 | } |
100 | | }; |
101 | | struct Relu { |
102 | 0 | inline TFloat operator()(TFloat x) const { |
103 | 0 | if (x <= 0) { |
104 | 0 | return 0; |
105 | 0 | } |
106 | 0 | return x; |
107 | 0 | } |
108 | | }; |
109 | | struct ReluPrime { |
110 | 0 | inline TFloat operator()(TFloat y) const { |
111 | 0 | return 0 < y ? 1 : 0; |
112 | 0 | } |
113 | | }; |
114 | | struct GFunc { |
115 | 10.8G | inline TFloat operator()(TFloat x) const { |
116 | 10.8G | return Tanh(x); |
117 | 10.8G | } |
118 | | }; |
119 | | struct GPrime { |
120 | 0 | inline TFloat operator()(TFloat y) const { |
121 | 0 | return 1 - y * y; |
122 | 0 | } |
123 | | }; |
124 | | struct ClipGFunc { |
125 | 0 | inline TFloat operator()(TFloat x) const { |
126 | 0 | if (x <= -1) { |
127 | 0 | return -1; |
128 | 0 | } |
129 | 0 | if (x >= 1) { |
130 | 0 | return 1; |
131 | 0 | } |
132 | 0 | return x; |
133 | 0 | } |
134 | | }; |
135 | | struct ClipGPrime { |
136 | 0 | inline TFloat operator()(TFloat y) const { |
137 | 0 | return -1 < y && y < 1 ? 1 : 0; |
138 | 0 | } |
139 | | }; |
140 | | struct HFunc { |
141 | 4.90G | inline TFloat operator()(TFloat x) const { |
142 | 4.90G | return Tanh(x); |
143 | 4.90G | } |
144 | | }; |
145 | | struct HPrime { |
146 | 0 | inline TFloat operator()(TFloat y) const { |
147 | 0 | TFloat u = Tanh(y); |
148 | 0 | return 1 - u * u; |
149 | 0 | } |
150 | | }; |
151 | | struct UnityFunc { |
152 | 0 | inline TFloat operator()(TFloat /*x*/) const { |
153 | 0 | return 1.0; |
154 | 0 | } |
155 | | }; |
156 | | struct IdentityFunc { |
157 | 0 | inline TFloat operator()(TFloat x) const { |
158 | 0 | return x; |
159 | 0 | } |
160 | | }; |
161 | | |
162 | | // Applies Func in-place to inout, of size n. |
163 | | template <class Func> |
164 | 569M | inline void FuncInplace(int n, TFloat *inout) { |
165 | 569M | Func f; |
166 | 26.0G | for (int i = 0; i < n; ++i) { |
167 | 25.5G | inout[i] = f(inout[i]); |
168 | 25.5G | } |
169 | 569M | } void tesseract::FuncInplace<tesseract::GFunc>(int, float*) Line | Count | Source | 164 | 419M | inline void FuncInplace(int n, TFloat *inout) { | 165 | 419M | Func f; | 166 | 11.2G | for (int i = 0; i < n; ++i) { | 167 | 10.8G | inout[i] = f(inout[i]); | 168 | 10.8G | } | 169 | 419M | } |
void tesseract::FuncInplace<tesseract::FFunc>(int, float*) Line | Count | Source | 164 | 149M | inline void FuncInplace(int n, TFloat *inout) { | 165 | 149M | Func f; | 166 | 14.8G | for (int i = 0; i < n; ++i) { | 167 | 14.7G | inout[i] = f(inout[i]); | 168 | 14.7G | } | 169 | 149M | } |
Unexecuted instantiation: void tesseract::FuncInplace<tesseract::ClipFFunc>(int, float*) Unexecuted instantiation: void tesseract::FuncInplace<tesseract::ClipGFunc>(int, float*) Unexecuted instantiation: void tesseract::FuncInplace<tesseract::Relu>(int, float*) |
170 | | // Applies Func to u and multiplies the result by v component-wise, |
171 | | // putting the product in out, all of size n. |
172 | | template <class Func> |
173 | 49.9M | inline void FuncMultiply(const TFloat *u, const TFloat *v, int n, TFloat *out) { |
174 | 49.9M | Func f; |
175 | 4.95G | for (int i = 0; i < n; ++i) { |
176 | 4.90G | out[i] = f(u[i]) * v[i]; |
177 | 4.90G | } |
178 | 49.9M | } |
179 | | // Applies the Softmax function in-place to inout, of size n. |
180 | | template <typename T> |
181 | 3.33M | inline void SoftmaxInPlace(int n, T *inout) { |
182 | 3.33M | if (n <= 0) { |
183 | 0 | return; |
184 | 0 | } |
185 | | // A limit on the negative range input to exp to guarantee non-zero output. |
186 | 3.33M | const T kMaxSoftmaxActivation = 86; |
187 | | |
188 | 3.33M | T max_output = inout[0]; |
189 | 369M | for (int i = 1; i < n; i++) { |
190 | 366M | T output = inout[i]; |
191 | 366M | if (output > max_output) { |
192 | 12.6M | max_output = output; |
193 | 12.6M | } |
194 | 366M | } |
195 | 3.33M | T prob_total = 0; |
196 | 373M | for (int i = 0; i < n; i++) { |
197 | 369M | T prob = inout[i] - max_output; |
198 | 369M | prob = std::exp(ClipToRange(prob, -kMaxSoftmaxActivation, static_cast<T>(0))); |
199 | 369M | prob_total += prob; |
200 | 369M | inout[i] = prob; |
201 | 369M | } |
202 | 3.33M | if (prob_total > 0) { |
203 | 373M | for (int i = 0; i < n; i++) { |
204 | 369M | inout[i] /= prob_total; |
205 | 369M | } |
206 | 3.33M | } |
207 | 3.33M | } |
208 | | |
209 | | // Copies n values of the given src vector to dest. |
210 | 0 | inline void CopyVector(unsigned n, const TFloat *src, TFloat *dest) { |
211 | 0 | memcpy(dest, src, n * sizeof(dest[0])); |
212 | 0 | } |
213 | | |
214 | | // Adds n values of the given src vector to dest. |
215 | 0 | inline void AccumulateVector(int n, const TFloat *src, TFloat *dest) { |
216 | 0 | for (int i = 0; i < n; ++i) { |
217 | 0 | dest[i] += src[i]; |
218 | 0 | } |
219 | 0 | } |
220 | | |
221 | | // Multiplies n values of inout in-place element-wise by the given src vector. |
222 | 49.9M | inline void MultiplyVectorsInPlace(int n, const TFloat *src, TFloat *inout) { |
223 | 4.95G | for (int i = 0; i < n; ++i) { |
224 | 4.90G | inout[i] *= src[i]; |
225 | 4.90G | } |
226 | 49.9M | } |
227 | | |
228 | | // Multiplies n values of u by v, element-wise, accumulating to out. |
229 | 49.9M | inline void MultiplyAccumulate(int n, const TFloat *u, const TFloat *v, TFloat *out) { |
230 | 4.95G | for (int i = 0; i < n; i++) { |
231 | 4.90G | out[i] += u[i] * v[i]; |
232 | 4.90G | } |
233 | 49.9M | } |
234 | | |
235 | | // Sums the given 5 n-vectors putting the result into sum. |
236 | | inline void SumVectors(int n, const TFloat *v1, const TFloat *v2, const TFloat *v3, |
237 | 0 | const TFloat *v4, const TFloat *v5, TFloat *sum) { |
238 | 0 | for (int i = 0; i < n; ++i) { |
239 | 0 | sum[i] = v1[i] + v2[i] + v3[i] + v4[i] + v5[i]; |
240 | 0 | } |
241 | 0 | } |
242 | | |
243 | | // Sets the given n-vector vec to 0. |
244 | | template <typename T> |
245 | 10.4M | inline void ZeroVector(unsigned n, T *vec) { |
246 | 10.4M | memset(vec, 0, n * sizeof(*vec)); |
247 | 10.4M | } Unexecuted instantiation: void tesseract::ZeroVector<signed char>(unsigned int, signed char*) void tesseract::ZeroVector<float>(unsigned int, float*) Line | Count | Source | 245 | 10.4M | inline void ZeroVector(unsigned n, T *vec) { | 246 | 10.4M | memset(vec, 0, n * sizeof(*vec)); | 247 | 10.4M | } |
|
248 | | |
249 | | // Clips the given vector vec, of size n to [lower, upper]. |
250 | | template <typename T> |
251 | 49.9M | inline void ClipVector(int n, T lower, T upper, T *vec) { |
252 | 4.95G | for (int i = 0; i < n; ++i) { |
253 | 4.90G | vec[i] = ClipToRange(vec[i], lower, upper); |
254 | 4.90G | } |
255 | 49.9M | } |
256 | | |
257 | | // Converts the given n-vector to a binary encoding of the maximum value, |
258 | | // encoded as vector of nf binary values. |
259 | 0 | inline void CodeInBinary(int n, int nf, TFloat *vec) { |
260 | 0 | if (nf <= 0 || n < nf) { |
261 | 0 | return; |
262 | 0 | } |
263 | 0 | int index = 0; |
264 | 0 | TFloat best_score = vec[0]; |
265 | 0 | for (int i = 1; i < n; ++i) { |
266 | 0 | if (vec[i] > best_score) { |
267 | 0 | best_score = vec[i]; |
268 | 0 | index = i; |
269 | 0 | } |
270 | 0 | } |
271 | 0 | int mask = 1; |
272 | 0 | for (int i = 0; i < nf; ++i, mask *= 2) { |
273 | 0 | vec[i] = (index & mask) ? 1.0 : 0.0; |
274 | 0 | } |
275 | 0 | } |
276 | | |
277 | | } // namespace tesseract. |
278 | | |
279 | | #endif // TESSERACT_LSTM_FUNCTIONS_H_ |