Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/tensorflow/python/keras/layers/advanced

3# Licensed under the Apache License, Version 2.0 (the "License");

4# you may not use this file except in compliance with the License.

5# You may obtain a copy of the License at

7# http://www.apache.org/licenses/LICENSE-2.0

9# Unless required by applicable law or agreed to in writing, software

10# distributed under the License is distributed on an "AS IS" BASIS,

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

12# See the License for the specific language governing permissions and

13# limitations under the License.

14# ==============================================================================

15"""Layers that act as activation functions."""

16# pylint: disable=g-classes-have-attributes

18from tensorflow.python.framework import dtypes

19from tensorflow.python.keras import backend

20from tensorflow.python.keras import constraints

21from tensorflow.python.keras import initializers

22from tensorflow.python.keras import regularizers

23from tensorflow.python.keras.engine.base_layer import Layer

24from tensorflow.python.keras.engine.input_spec import InputSpec

25from tensorflow.python.keras.utils import tf_utils

26from tensorflow.python.ops import math_ops

27from tensorflow.python.util.tf_export import keras_export

30def get_globals():

31 return globals()

34@keras_export('keras.layers.LeakyReLU')

35class LeakyReLU(Layer):

36 """Leaky version of a Rectified Linear Unit.

38 It allows a small gradient when the unit is not active:

40 ```

41 f(x) = alpha * x if x < 0

42 f(x) = x if x >= 0

43 ```

45 Usage:

47 >>> layer = tf.keras.layers.LeakyReLU()

48 >>> output = layer([-3.0, -1.0, 0.0, 2.0])

49 >>> list(output.numpy())

50 [-0.9, -0.3, 0.0, 2.0]

51 >>> layer = tf.keras.layers.LeakyReLU(alpha=0.1)

52 >>> output = layer([-3.0, -1.0, 0.0, 2.0])

53 >>> list(output.numpy())

54 [-0.3, -0.1, 0.0, 2.0]

56 Input shape:

57 Arbitrary. Use the keyword argument `input_shape`

58 (tuple of integers, does not include the batch axis)

59 when using this layer as the first layer in a model.

61 Output shape:

62 Same shape as the input.

64 Args:

65 alpha: Float >= 0. Negative slope coefficient. Default to 0.3.

67 """

69 def __init__(self, alpha=0.3, **kwargs):

70 super(LeakyReLU, self).__init__(**kwargs)

71 if alpha is None:

72 raise ValueError('The alpha value of a Leaky ReLU layer '

73 'cannot be None, needs a float. '

74 'Got %s' % alpha)

75 self.supports_masking = True

76 self.alpha = backend.cast_to_floatx(alpha)

78 def call(self, inputs):

79 return backend.relu(inputs, alpha=self.alpha)

81 def get_config(self):

82 config = {'alpha': float(self.alpha)}

83 base_config = super(LeakyReLU, self).get_config()

84 return dict(list(base_config.items()) + list(config.items()))

86 @tf_utils.shape_type_conversion

87 def compute_output_shape(self, input_shape):

88 return input_shape

91@keras_export('keras.layers.PReLU')

92class PReLU(Layer):

93 """Parametric Rectified Linear Unit.

95 It follows:

97 ```

98 f(x) = alpha * x for x < 0

99 f(x) = x for x >= 0

100 ```

101

102 where `alpha` is a learned array with the same shape as x.

103

104 Input shape:

105 Arbitrary. Use the keyword argument `input_shape`

106 (tuple of integers, does not include the samples axis)

107 when using this layer as the first layer in a model.

108

109 Output shape:

110 Same shape as the input.

111

112 Args:

113 alpha_initializer: Initializer function for the weights.

114 alpha_regularizer: Regularizer for the weights.

115 alpha_constraint: Constraint for the weights.

116 shared_axes: The axes along which to share learnable

117 parameters for the activation function.

118 For example, if the incoming feature maps

119 are from a 2D convolution

120 with output shape `(batch, height, width, channels)`,

121 and you wish to share parameters across space

122 so that each filter only has one set of parameters,

123 set `shared_axes=[1, 2]`.

124 """

125

126 def __init__(self,

127 alpha_initializer='zeros',

128 alpha_regularizer=None,

129 alpha_constraint=None,

130 shared_axes=None,

131 **kwargs):

132 super(PReLU, self).__init__(**kwargs)

133 self.supports_masking = True

134 self.alpha_initializer = initializers.get(alpha_initializer)

135 self.alpha_regularizer = regularizers.get(alpha_regularizer)

136 self.alpha_constraint = constraints.get(alpha_constraint)

137 if shared_axes is None:

138 self.shared_axes = None

139 elif not isinstance(shared_axes, (list, tuple)):

140 self.shared_axes = [shared_axes]

141 else:

142 self.shared_axes = list(shared_axes)

143

144 @tf_utils.shape_type_conversion

145 def build(self, input_shape):

146 param_shape = list(input_shape[1:])

147 if self.shared_axes is not None:

148 for i in self.shared_axes:

149 param_shape[i - 1] = 1

150 self.alpha = self.add_weight(

151 shape=param_shape,

152 name='alpha',

153 initializer=self.alpha_initializer,

154 regularizer=self.alpha_regularizer,

155 constraint=self.alpha_constraint)

156 # Set input spec

157 axes = {}

158 if self.shared_axes:

159 for i in range(1, len(input_shape)):

160 if i not in self.shared_axes:

161 axes[i] = input_shape[i]

162 self.input_spec = InputSpec(ndim=len(input_shape), axes=axes)

163 self.built = True

164

165 def call(self, inputs):

166 pos = backend.relu(inputs)

167 neg = -self.alpha * backend.relu(-inputs)

168 return pos + neg

169

170 def get_config(self):

171 config = {

172 'alpha_initializer': initializers.serialize(self.alpha_initializer),

173 'alpha_regularizer': regularizers.serialize(self.alpha_regularizer),

174 'alpha_constraint': constraints.serialize(self.alpha_constraint),

175 'shared_axes': self.shared_axes

176 }

177 base_config = super(PReLU, self).get_config()

178 return dict(list(base_config.items()) + list(config.items()))

179

180 @tf_utils.shape_type_conversion

181 def compute_output_shape(self, input_shape):

182 return input_shape

183

184

185@keras_export('keras.layers.ELU')

186class ELU(Layer):

187 """Exponential Linear Unit.

188

189 It follows:

190

191 ```

192 f(x) = alpha * (exp(x) - 1.) for x < 0

193 f(x) = x for x >= 0

194 ```

195

196 Input shape:

197 Arbitrary. Use the keyword argument `input_shape`

198 (tuple of integers, does not include the samples axis)

199 when using this layer as the first layer in a model.

200

201 Output shape:

202 Same shape as the input.

203

204 Args:

205 alpha: Scale for the negative factor.

206 """

207

208 def __init__(self, alpha=1.0, **kwargs):

209 super(ELU, self).__init__(**kwargs)

210 if alpha is None:

211 raise ValueError('Alpha of an ELU layer cannot be None, '

212 'requires a float. Got %s' % alpha)

213 self.supports_masking = True

214 self.alpha = backend.cast_to_floatx(alpha)

215

216 def call(self, inputs):

217 return backend.elu(inputs, self.alpha)

218

219 def get_config(self):

220 config = {'alpha': float(self.alpha)}

221 base_config = super(ELU, self).get_config()

222 return dict(list(base_config.items()) + list(config.items()))

223

224 @tf_utils.shape_type_conversion

225 def compute_output_shape(self, input_shape):

226 return input_shape

227

228

229@keras_export('keras.layers.ThresholdedReLU')

230class ThresholdedReLU(Layer):

231 """Thresholded Rectified Linear Unit.

232

233 It follows:

234

235 ```

236 f(x) = x for x > theta

237 f(x) = 0 otherwise`

238 ```

239

240 Input shape:

241 Arbitrary. Use the keyword argument `input_shape`

242 (tuple of integers, does not include the samples axis)

243 when using this layer as the first layer in a model.

244

245 Output shape:

246 Same shape as the input.

247

248 Args:

249 theta: Float >= 0. Threshold location of activation.

250 """

251

252 def __init__(self, theta=1.0, **kwargs):

253 super(ThresholdedReLU, self).__init__(**kwargs)

254 if theta is None:

255 raise ValueError('Theta of a Thresholded ReLU layer cannot be '

256 'None, requires a float. Got %s' % theta)

257 if theta < 0:

258 raise ValueError('The theta value of a Thresholded ReLU layer '

259 'should be >=0, got %s' % theta)

260 self.supports_masking = True

261 self.theta = backend.cast_to_floatx(theta)

262

263 def call(self, inputs):

264 theta = math_ops.cast(self.theta, inputs.dtype)

265 return inputs * math_ops.cast(math_ops.greater(inputs, theta), inputs.dtype)

266

267 def get_config(self):

268 config = {'theta': float(self.theta)}

269 base_config = super(ThresholdedReLU, self).get_config()

270 return dict(list(base_config.items()) + list(config.items()))

271

272 @tf_utils.shape_type_conversion

273 def compute_output_shape(self, input_shape):

274 return input_shape

275

276

277def _large_compatible_negative(tensor_type):

278 """Large negative number as Tensor.

279

280 This function is necessary because the standard value for epsilon

281 in this module (-1e9) cannot be represented using tf.float16

282

283 Args:

284 tensor_type: a dtype to determine the type.

285

286 Returns:

287 a large negative number.

288 """

289 if tensor_type == dtypes.float16:

290 return dtypes.float16.min

291 return -1e9

292

293

294@keras_export('keras.layers.Softmax')

295class Softmax(Layer):

296 """Softmax activation function.

297

298 Example without mask:

299

300 >>> inp = np.asarray([1., 2., 1.])

301 >>> layer = tf.keras.layers.Softmax()

302 >>> layer(inp).numpy()

303 array([0.21194157, 0.5761169 , 0.21194157], dtype=float32)

304 >>> mask = np.asarray([True, False, True], dtype=bool)

305 >>> layer(inp, mask).numpy()

306 array([0.5, 0. , 0.5], dtype=float32)

307

308 Input shape:

309 Arbitrary. Use the keyword argument `input_shape`

310 (tuple of integers, does not include the samples axis)

311 when using this layer as the first layer in a model.

312

313 Output shape:

314 Same shape as the input.

315

316 Args:

317 axis: Integer, or list of Integers, axis along which the softmax

318 normalization is applied.

319 Call arguments:

320 inputs: The inputs, or logits to the softmax layer.

321 mask: A boolean mask of the same shape as `inputs`. Defaults to `None`. The

322 mask specifies 1 to keep and 0 to mask.

323

324 Returns:

325 softmaxed output with the same shape as `inputs`.

326 """

327

328 def __init__(self, axis=-1, **kwargs):

329 super(Softmax, self).__init__(**kwargs)

330 self.supports_masking = True

331 self.axis = axis

332

333 def call(self, inputs, mask=None):

334 if mask is not None:

335 # Since mask is 1.0 for positions we want to keep and 0.0 for

336 # masked positions, this operation will create a tensor which is 0.0 for

337 # positions we want to attend and -1e.9 for masked positions.

338 adder = (1.0 - math_ops.cast(mask, inputs.dtype)) * (

339 _large_compatible_negative(inputs.dtype))

340

341 # Since we are adding it to the raw scores before the softmax, this is

342 # effectively the same as removing these entirely.

343 inputs += adder

344 if isinstance(self.axis, (tuple, list)):

345 if len(self.axis) > 1:

346 return math_ops.exp(inputs - math_ops.reduce_logsumexp(

347 inputs, axis=self.axis, keepdims=True))

348 else:

349 return backend.softmax(inputs, axis=self.axis[0])

350 return backend.softmax(inputs, axis=self.axis)

351

352 def get_config(self):

353 config = {'axis': self.axis}

354 base_config = super(Softmax, self).get_config()

355 return dict(list(base_config.items()) + list(config.items()))

356

357 @tf_utils.shape_type_conversion

358 def compute_output_shape(self, input_shape):

359 return input_shape

360

361

362@keras_export('keras.layers.ReLU')

363class ReLU(Layer):

364 """Rectified Linear Unit activation function.

365

366 With default values, it returns element-wise `max(x, 0)`.

367

368 Otherwise, it follows:

369

370 ```

371 f(x) = max_value if x >= max_value

372 f(x) = x if threshold <= x < max_value

373 f(x) = negative_slope * (x - threshold) otherwise

374 ```

375

376 Usage:

377

378 >>> layer = tf.keras.layers.ReLU()

379 >>> output = layer([-3.0, -1.0, 0.0, 2.0])

380 >>> list(output.numpy())

381 [0.0, 0.0, 0.0, 2.0]

382 >>> layer = tf.keras.layers.ReLU(max_value=1.0)

383 >>> output = layer([-3.0, -1.0, 0.0, 2.0])

384 >>> list(output.numpy())

385 [0.0, 0.0, 0.0, 1.0]

386 >>> layer = tf.keras.layers.ReLU(negative_slope=1.0)

387 >>> output = layer([-3.0, -1.0, 0.0, 2.0])

388 >>> list(output.numpy())

389 [-3.0, -1.0, 0.0, 2.0]

390 >>> layer = tf.keras.layers.ReLU(threshold=1.5)

391 >>> output = layer([-3.0, -1.0, 1.0, 2.0])

392 >>> list(output.numpy())

393 [0.0, 0.0, 0.0, 2.0]

394

395 Input shape:

396 Arbitrary. Use the keyword argument `input_shape`

397 (tuple of integers, does not include the batch axis)

398 when using this layer as the first layer in a model.

399

400 Output shape:

401 Same shape as the input.

402

403 Args:

404 max_value: Float >= 0. Maximum activation value. Default to None, which

405 means unlimited.

406 negative_slope: Float >= 0. Negative slope coefficient. Default to 0.

407 threshold: Float >= 0. Threshold value for thresholded activation. Default

408 to 0.

409 """

410

411 def __init__(self, max_value=None, negative_slope=0, threshold=0, **kwargs):

412 super(ReLU, self).__init__(**kwargs)

413 if max_value is not None and max_value < 0.:

414 raise ValueError('max_value of a ReLU layer cannot be a negative '

415 'value. Got: %s' % max_value)

416 if negative_slope is None or negative_slope < 0.:

417 raise ValueError('negative_slope of a ReLU layer cannot be a negative '

418 'value. Got: %s' % negative_slope)

419 if threshold is None or threshold < 0.:

420 raise ValueError('threshold of a ReLU layer cannot be a negative '

421 'value. Got: %s' % threshold)

422

423 self.supports_masking = True

424 if max_value is not None:

425 max_value = backend.cast_to_floatx(max_value)

426 self.max_value = max_value

427 self.negative_slope = backend.cast_to_floatx(negative_slope)

428 self.threshold = backend.cast_to_floatx(threshold)

429

430 def call(self, inputs):

431 # alpha is used for leaky relu slope in activations instead of

432 # negative_slope.

433 return backend.relu(inputs,

434 alpha=self.negative_slope,

435 max_value=self.max_value,

436 threshold=self.threshold)

437

438 def get_config(self):

439 config = {

440 'max_value': self.max_value,

441 'negative_slope': self.negative_slope,

442 'threshold': self.threshold

443 }

444 base_config = super(ReLU, self).get_config()

445 return dict(list(base_config.items()) + list(config.items()))

446

447 @tf_utils.shape_type_conversion

448 def compute_output_shape(self, input_shape):

449 return input_shape

Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/tensorflow/python/keras/layers/advanced_activations.py: 37%

156 statements