Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/tensorflow/python/keras/layers/core.py: 29%
672 statements
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-03 07:57 +0000
1# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""Core Keras layers."""
17import copy
18import functools
19import operator
20import sys
21import textwrap
22import types as python_types
23import warnings
25import numpy as np
27from tensorflow.python.eager import backprop
28from tensorflow.python.eager import context
29from tensorflow.python.framework import constant_op
30from tensorflow.python.framework import dtypes
31from tensorflow.python.framework import sparse_tensor
32from tensorflow.python.framework import tensor_conversion
33from tensorflow.python.framework import tensor_shape
34from tensorflow.python.keras import activations
35from tensorflow.python.keras import backend as K
36from tensorflow.python.keras import constraints
37from tensorflow.python.keras import initializers
38from tensorflow.python.keras import regularizers
39from tensorflow.python.keras.engine import keras_tensor
40from tensorflow.python.keras.engine.base_layer import Layer
41from tensorflow.python.keras.engine.input_spec import InputSpec
42from tensorflow.python.keras.utils import control_flow_util
43from tensorflow.python.keras.utils import conv_utils
44from tensorflow.python.keras.utils import generic_utils
45from tensorflow.python.keras.utils import tf_inspect
46from tensorflow.python.keras.utils import tf_utils
47from tensorflow.python.ops import array_ops
48from tensorflow.python.ops import embedding_ops
49from tensorflow.python.ops import gen_math_ops
50from tensorflow.python.ops import math_ops
51from tensorflow.python.ops import nn
52from tensorflow.python.ops import nn_ops
53from tensorflow.python.ops import sparse_ops
54from tensorflow.python.ops import standard_ops
55from tensorflow.python.ops import variable_scope
56from tensorflow.python.ops.ragged import ragged_getitem
57from tensorflow.python.ops.ragged import ragged_tensor
58from tensorflow.python.platform import tf_logging
59from tensorflow.python.trackable import base as trackable
60from tensorflow.python.util import dispatch
61from tensorflow.python.util import nest
62from tensorflow.python.util import tf_decorator
63from tensorflow.python.util.tf_export import get_canonical_name_for_symbol
64from tensorflow.python.util.tf_export import get_symbol_from_name
65from tensorflow.python.util.tf_export import keras_export
68# pylint: disable=g-classes-have-attributes
69@keras_export('keras.layers.Masking')
70class Masking(Layer):
71 """Masks a sequence by using a mask value to skip timesteps.
73 For each timestep in the input tensor (dimension #1 in the tensor),
74 if all values in the input tensor at that timestep
75 are equal to `mask_value`, then the timestep will be masked (skipped)
76 in all downstream layers (as long as they support masking).
78 If any downstream layer does not support masking yet receives such
79 an input mask, an exception will be raised.
81 Example:
83 Consider a Numpy data array `x` of shape `(samples, timesteps, features)`,
84 to be fed to an LSTM layer. You want to mask timestep #3 and #5 because you
85 lack data for these timesteps. You can:
87 - Set `x[:, 3, :] = 0.` and `x[:, 5, :] = 0.`
88 - Insert a `Masking` layer with `mask_value=0.` before the LSTM layer:
90 ```python
91 samples, timesteps, features = 32, 10, 8
92 inputs = np.random.random([samples, timesteps, features]).astype(np.float32)
93 inputs[:, 3, :] = 0.
94 inputs[:, 5, :] = 0.
96 model = tf.keras.models.Sequential()
97 model.add(tf.keras.layers.Masking(mask_value=0.,
98 input_shape=(timesteps, features)))
99 model.add(tf.keras.layers.LSTM(32))
101 output = model(inputs)
102 # The time step 3 and 5 will be skipped from LSTM calculation.
103 ```
105 See [the masking and padding guide](
106 https://www.tensorflow.org/guide/keras/masking_and_padding)
107 for more details.
108 """
110 def __init__(self, mask_value=0., **kwargs):
111 super(Masking, self).__init__(**kwargs)
112 self.supports_masking = True
113 self.mask_value = mask_value
114 self._compute_output_and_mask_jointly = True
116 def compute_mask(self, inputs, mask=None):
117 return K.any(math_ops.not_equal(inputs, self.mask_value), axis=-1)
119 def call(self, inputs):
120 boolean_mask = K.any(
121 math_ops.not_equal(inputs, self.mask_value), axis=-1, keepdims=True)
122 outputs = inputs * math_ops.cast(boolean_mask, inputs.dtype)
123 # Compute the mask and outputs simultaneously.
124 outputs._keras_mask = array_ops.squeeze(boolean_mask, axis=-1) # pylint: disable=protected-access
125 return outputs
127 def compute_output_shape(self, input_shape):
128 return input_shape
130 def get_config(self):
131 config = {'mask_value': self.mask_value}
132 base_config = super(Masking, self).get_config()
133 return dict(list(base_config.items()) + list(config.items()))
136@keras_export('keras.layers.Dropout')
137class Dropout(Layer):
138 """Applies Dropout to the input.
140 The Dropout layer randomly sets input units to 0 with a frequency of `rate`
141 at each step during training time, which helps prevent overfitting.
142 Inputs not set to 0 are scaled up by 1/(1 - rate) such that the sum over
143 all inputs is unchanged.
145 Note that the Dropout layer only applies when `training` is set to True
146 such that no values are dropped during inference. When using `model.fit`,
147 `training` will be appropriately set to True automatically, and in other
148 contexts, you can set the kwarg explicitly to True when calling the layer.
150 (This is in contrast to setting `trainable=False` for a Dropout layer.
151 `trainable` does not affect the layer's behavior, as Dropout does
152 not have any variables/weights that can be frozen during training.)
154 >>> tf.random.set_seed(0)
155 >>> layer = tf.keras.layers.Dropout(.2, input_shape=(2,))
156 >>> data = np.arange(10).reshape(5, 2).astype(np.float32)
157 >>> print(data)
158 [[0. 1.]
159 [2. 3.]
160 [4. 5.]
161 [6. 7.]
162 [8. 9.]]
163 >>> outputs = layer(data, training=True)
164 >>> print(outputs)
165 tf.Tensor(
166 [[ 0. 1.25]
167 [ 2.5 3.75]
168 [ 5. 6.25]
169 [ 7.5 8.75]
170 [10. 0. ]], shape=(5, 2), dtype=float32)
172 Args:
173 rate: Float between 0 and 1. Fraction of the input units to drop.
174 noise_shape: 1D integer tensor representing the shape of the
175 binary dropout mask that will be multiplied with the input.
176 For instance, if your inputs have shape
177 `(batch_size, timesteps, features)` and
178 you want the dropout mask to be the same for all timesteps,
179 you can use `noise_shape=(batch_size, 1, features)`.
180 seed: A Python integer to use as random seed.
182 Call arguments:
183 inputs: Input tensor (of any rank).
184 training: Python boolean indicating whether the layer should behave in
185 training mode (adding dropout) or in inference mode (doing nothing).
186 """
188 def __init__(self, rate, noise_shape=None, seed=None, **kwargs):
189 super(Dropout, self).__init__(**kwargs)
190 if isinstance(rate, (int, float)) and not 0 <= rate <= 1:
191 raise ValueError(f'Invalid value {rate} received for '
192 f'`rate`, expected a value between 0 and 1.')
193 self.rate = rate
194 self.noise_shape = noise_shape
195 self.seed = seed
196 self.supports_masking = True
198 def _get_noise_shape(self, inputs):
199 # Subclasses of `Dropout` may implement `_get_noise_shape(self, inputs)`,
200 # which will override `self.noise_shape`, and allows for custom noise
201 # shapes with dynamically sized inputs.
202 if self.noise_shape is None:
203 return None
205 concrete_inputs_shape = array_ops.shape(inputs)
206 noise_shape = []
207 for i, value in enumerate(self.noise_shape):
208 noise_shape.append(concrete_inputs_shape[i] if value is None else value)
209 return tensor_conversion.convert_to_tensor_v2_with_dispatch(noise_shape)
211 def call(self, inputs, training=None):
212 if training is None:
213 training = K.learning_phase()
215 def dropped_inputs():
216 return nn.dropout(
217 inputs,
218 noise_shape=self._get_noise_shape(inputs),
219 seed=self.seed,
220 rate=self.rate)
222 output = control_flow_util.smart_cond(training, dropped_inputs,
223 lambda: array_ops.identity(inputs))
224 return output
226 def compute_output_shape(self, input_shape):
227 return input_shape
229 def get_config(self):
230 config = {
231 'rate': self.rate,
232 'noise_shape': self.noise_shape,
233 'seed': self.seed
234 }
235 base_config = super(Dropout, self).get_config()
236 return dict(list(base_config.items()) + list(config.items()))
239@keras_export('keras.layers.SpatialDropout1D')
240class SpatialDropout1D(Dropout):
241 """Spatial 1D version of Dropout.
243 This version performs the same function as Dropout, however, it drops
244 entire 1D feature maps instead of individual elements. If adjacent frames
245 within feature maps are strongly correlated (as is normally the case in
246 early convolution layers) then regular dropout will not regularize the
247 activations and will otherwise just result in an effective learning rate
248 decrease. In this case, SpatialDropout1D will help promote independence
249 between feature maps and should be used instead.
251 Args:
252 rate: Float between 0 and 1. Fraction of the input units to drop.
254 Call arguments:
255 inputs: A 3D tensor.
256 training: Python boolean indicating whether the layer should behave in
257 training mode (adding dropout) or in inference mode (doing nothing).
259 Input shape:
260 3D tensor with shape:
261 `(samples, timesteps, channels)`
263 Output shape:
264 Same as input.
266 References:
267 - [Efficient Object Localization Using Convolutional
268 Networks](https://arxiv.org/abs/1411.4280)
269 """
271 def __init__(self, rate, **kwargs):
272 super(SpatialDropout1D, self).__init__(rate, **kwargs)
273 self.input_spec = InputSpec(ndim=3)
275 def _get_noise_shape(self, inputs):
276 input_shape = array_ops.shape(inputs)
277 noise_shape = (input_shape[0], 1, input_shape[2])
278 return noise_shape
281@keras_export('keras.layers.SpatialDropout2D')
282class SpatialDropout2D(Dropout):
283 """Spatial 2D version of Dropout.
285 This version performs the same function as Dropout, however, it drops
286 entire 2D feature maps instead of individual elements. If adjacent pixels
287 within feature maps are strongly correlated (as is normally the case in
288 early convolution layers) then regular dropout will not regularize the
289 activations and will otherwise just result in an effective learning rate
290 decrease. In this case, SpatialDropout2D will help promote independence
291 between feature maps and should be used instead.
293 Args:
294 rate: Float between 0 and 1. Fraction of the input units to drop.
295 data_format: 'channels_first' or 'channels_last'.
296 In 'channels_first' mode, the channels dimension
297 (the depth) is at index 1,
298 in 'channels_last' mode is it at index 3.
299 It defaults to the `image_data_format` value found in your
300 Keras config file at `~/.keras/keras.json`.
301 If you never set it, then it will be "channels_last".
303 Call arguments:
304 inputs: A 4D tensor.
305 training: Python boolean indicating whether the layer should behave in
306 training mode (adding dropout) or in inference mode (doing nothing).
308 Input shape:
309 4D tensor with shape:
310 `(samples, channels, rows, cols)` if data_format='channels_first'
311 or 4D tensor with shape:
312 `(samples, rows, cols, channels)` if data_format='channels_last'.
314 Output shape:
315 Same as input.
317 References:
318 - [Efficient Object Localization Using Convolutional
319 Networks](https://arxiv.org/abs/1411.4280)
320 """
322 def __init__(self, rate, data_format=None, **kwargs):
323 super(SpatialDropout2D, self).__init__(rate, **kwargs)
324 if data_format is None:
325 data_format = K.image_data_format()
326 if data_format not in {'channels_last', 'channels_first'}:
327 raise ValueError('data_format must be in '
328 '{"channels_last", "channels_first"}')
329 self.data_format = data_format
330 self.input_spec = InputSpec(ndim=4)
332 def _get_noise_shape(self, inputs):
333 input_shape = array_ops.shape(inputs)
334 if self.data_format == 'channels_first':
335 return (input_shape[0], input_shape[1], 1, 1)
336 elif self.data_format == 'channels_last':
337 return (input_shape[0], 1, 1, input_shape[3])
340@keras_export('keras.layers.SpatialDropout3D')
341class SpatialDropout3D(Dropout):
342 """Spatial 3D version of Dropout.
344 This version performs the same function as Dropout, however, it drops
345 entire 3D feature maps instead of individual elements. If adjacent voxels
346 within feature maps are strongly correlated (as is normally the case in
347 early convolution layers) then regular dropout will not regularize the
348 activations and will otherwise just result in an effective learning rate
349 decrease. In this case, SpatialDropout3D will help promote independence
350 between feature maps and should be used instead.
352 Args:
353 rate: Float between 0 and 1. Fraction of the input units to drop.
354 data_format: 'channels_first' or 'channels_last'.
355 In 'channels_first' mode, the channels dimension (the depth)
356 is at index 1, in 'channels_last' mode is it at index 4.
357 It defaults to the `image_data_format` value found in your
358 Keras config file at `~/.keras/keras.json`.
359 If you never set it, then it will be "channels_last".
361 Call arguments:
362 inputs: A 5D tensor.
363 training: Python boolean indicating whether the layer should behave in
364 training mode (adding dropout) or in inference mode (doing nothing).
366 Input shape:
367 5D tensor with shape:
368 `(samples, channels, dim1, dim2, dim3)` if data_format='channels_first'
369 or 5D tensor with shape:
370 `(samples, dim1, dim2, dim3, channels)` if data_format='channels_last'.
372 Output shape:
373 Same as input.
375 References:
376 - [Efficient Object Localization Using Convolutional
377 Networks](https://arxiv.org/abs/1411.4280)
378 """
380 def __init__(self, rate, data_format=None, **kwargs):
381 super(SpatialDropout3D, self).__init__(rate, **kwargs)
382 if data_format is None:
383 data_format = K.image_data_format()
384 if data_format not in {'channels_last', 'channels_first'}:
385 raise ValueError('data_format must be in '
386 '{"channels_last", "channels_first"}')
387 self.data_format = data_format
388 self.input_spec = InputSpec(ndim=5)
390 def _get_noise_shape(self, inputs):
391 input_shape = array_ops.shape(inputs)
392 if self.data_format == 'channels_first':
393 return (input_shape[0], input_shape[1], 1, 1, 1)
394 elif self.data_format == 'channels_last':
395 return (input_shape[0], 1, 1, 1, input_shape[4])
398@keras_export('keras.layers.Activation')
399class Activation(Layer):
400 """Applies an activation function to an output.
402 Args:
403 activation: Activation function, such as `tf.nn.relu`, or string name of
404 built-in activation function, such as "relu".
406 Usage:
408 >>> layer = tf.keras.layers.Activation('relu')
409 >>> output = layer([-3.0, -1.0, 0.0, 2.0])
410 >>> list(output.numpy())
411 [0.0, 0.0, 0.0, 2.0]
412 >>> layer = tf.keras.layers.Activation(tf.nn.relu)
413 >>> output = layer([-3.0, -1.0, 0.0, 2.0])
414 >>> list(output.numpy())
415 [0.0, 0.0, 0.0, 2.0]
417 Input shape:
418 Arbitrary. Use the keyword argument `input_shape`
419 (tuple of integers, does not include the batch axis)
420 when using this layer as the first layer in a model.
422 Output shape:
423 Same shape as input.
424 """
426 def __init__(self, activation, **kwargs):
427 super(Activation, self).__init__(**kwargs)
428 self.supports_masking = True
429 self.activation = activations.get(activation)
431 def call(self, inputs):
432 return self.activation(inputs)
434 def compute_output_shape(self, input_shape):
435 return input_shape
437 def get_config(self):
438 config = {'activation': activations.serialize(self.activation)}
439 base_config = super(Activation, self).get_config()
440 return dict(list(base_config.items()) + list(config.items()))
443@keras_export('keras.layers.Reshape')
444class Reshape(Layer):
445 """Layer that reshapes inputs into the given shape.
447 Input shape:
448 Arbitrary, although all dimensions in the input shape must be known/fixed.
449 Use the keyword argument `input_shape` (tuple of integers, does not include
450 the samples/batch size axis) when using this layer as the first layer
451 in a model.
453 Output shape:
454 `(batch_size,) + target_shape`
456 Example:
458 >>> # as first layer in a Sequential model
459 >>> model = tf.keras.Sequential()
460 >>> model.add(tf.keras.layers.Reshape((3, 4), input_shape=(12,)))
461 >>> # model.output_shape == (None, 3, 4), `None` is the batch size.
462 >>> model.output_shape
463 (None, 3, 4)
465 >>> # as intermediate layer in a Sequential model
466 >>> model.add(tf.keras.layers.Reshape((6, 2)))
467 >>> model.output_shape
468 (None, 6, 2)
470 >>> # also supports shape inference using `-1` as dimension
471 >>> model.add(tf.keras.layers.Reshape((-1, 2, 2)))
472 >>> model.output_shape
473 (None, 3, 2, 2)
474 """
476 def __init__(self, target_shape, **kwargs):
477 """Creates a `tf.keras.layers.Reshape` layer instance.
479 Args:
480 target_shape: Target shape. Tuple of integers, does not include the
481 samples dimension (batch size).
482 **kwargs: Any additional layer keyword arguments.
483 """
484 super(Reshape, self).__init__(**kwargs)
485 self.target_shape = tuple(target_shape)
487 def _fix_unknown_dimension(self, input_shape, output_shape):
488 """Find and replace a missing dimension in an output shape.
490 This is a near direct port of the internal Numpy function
491 `_fix_unknown_dimension` in `numpy/core/src/multiarray/shape.c`
493 Args:
494 input_shape: Shape of array being reshaped
495 output_shape: Desired shape of the array with at most
496 a single -1 which indicates a dimension that should be
497 derived from the input shape.
499 Returns:
500 The new output shape with a -1 replaced with its computed value.
502 Raises:
503 ValueError: If the total array size of the output_shape is
504 different than the input_shape, or more than one unknown dimension
505 is specified.
506 """
507 output_shape = list(output_shape)
508 msg = ('total size of new array must be unchanged, '
509 'input_shape = {}, output_shape = {}'
510 .format(input_shape, output_shape))
512 known, unknown = 1, None
513 for index, dim in enumerate(output_shape):
514 if dim < 0:
515 if unknown is None:
516 unknown = index
517 else:
518 raise ValueError('Can only specify one unknown dimension.')
519 else:
520 known *= dim
522 original = np.prod(input_shape, dtype=int)
523 if unknown is not None:
524 if known == 0 or original % known != 0:
525 raise ValueError(msg)
526 output_shape[unknown] = original // known
527 elif original != known:
528 raise ValueError(msg)
529 return output_shape
531 def compute_output_shape(self, input_shape):
532 input_shape = tensor_shape.TensorShape(input_shape).as_list()
533 if None in input_shape[1:]:
534 output_shape = [input_shape[0]]
535 # input shape (partially) unknown? replace -1's with None's
536 output_shape += tuple(s if s != -1 else None for s in self.target_shape)
537 else:
538 output_shape = [input_shape[0]]
539 output_shape += self._fix_unknown_dimension(input_shape[1:],
540 self.target_shape)
541 return tensor_shape.TensorShape(output_shape)
543 def call(self, inputs):
544 result = array_ops.reshape(
545 inputs, (array_ops.shape(inputs)[0],) + self.target_shape)
546 if not context.executing_eagerly():
547 # Set the static shape for the result since it might lost during array_ops
548 # reshape, eg, some `None` dim in the result could be inferred.
549 result.set_shape(self.compute_output_shape(inputs.shape))
550 return result
552 def get_config(self):
553 config = {'target_shape': self.target_shape}
554 base_config = super(Reshape, self).get_config()
555 return dict(list(base_config.items()) + list(config.items()))
558@keras_export('keras.layers.Permute')
559class Permute(Layer):
560 """Permutes the dimensions of the input according to a given pattern.
562 Useful e.g. connecting RNNs and convnets.
564 Example:
566 ```python
567 model = Sequential()
568 model.add(Permute((2, 1), input_shape=(10, 64)))
569 # now: model.output_shape == (None, 64, 10)
570 # note: `None` is the batch dimension
571 ```
573 Args:
574 dims: Tuple of integers. Permutation pattern does not include the
575 samples dimension. Indexing starts at 1.
576 For instance, `(2, 1)` permutes the first and second dimensions
577 of the input.
579 Input shape:
580 Arbitrary. Use the keyword argument `input_shape`
581 (tuple of integers, does not include the samples axis)
582 when using this layer as the first layer in a model.
584 Output shape:
585 Same as the input shape, but with the dimensions re-ordered according
586 to the specified pattern.
587 """
589 def __init__(self, dims, **kwargs):
590 super(Permute, self).__init__(**kwargs)
591 self.dims = tuple(dims)
592 if sorted(dims) != list(range(1, len(dims) + 1)):
593 raise ValueError(
594 'Invalid permutation `dims` for Permute Layer: %s. '
595 'The set of indices in `dims` must be consecutive and start from 1.' %
596 (dims,))
597 self.input_spec = InputSpec(ndim=len(self.dims) + 1)
599 def compute_output_shape(self, input_shape):
600 input_shape = tensor_shape.TensorShape(input_shape).as_list()
601 output_shape = copy.copy(input_shape)
602 for i, dim in enumerate(self.dims):
603 target_dim = input_shape[dim]
604 output_shape[i + 1] = target_dim
605 return tensor_shape.TensorShape(output_shape)
607 def call(self, inputs):
608 return array_ops.transpose(inputs, perm=(0,) + self.dims)
610 def get_config(self):
611 config = {'dims': self.dims}
612 base_config = super(Permute, self).get_config()
613 return dict(list(base_config.items()) + list(config.items()))
616@keras_export('keras.layers.Flatten')
617class Flatten(Layer):
618 """Flattens the input. Does not affect the batch size.
620 Note: If inputs are shaped `(batch,)` without a feature axis, then
621 flattening adds an extra channel dimension and output shape is `(batch, 1)`.
623 Args:
624 data_format: A string,
625 one of `channels_last` (default) or `channels_first`.
626 The ordering of the dimensions in the inputs.
627 `channels_last` corresponds to inputs with shape
628 `(batch, ..., channels)` while `channels_first` corresponds to
629 inputs with shape `(batch, channels, ...)`.
630 It defaults to the `image_data_format` value found in your
631 Keras config file at `~/.keras/keras.json`.
632 If you never set it, then it will be "channels_last".
634 Example:
636 >>> model = tf.keras.Sequential()
637 >>> model.add(tf.keras.layers.Conv2D(64, 3, 3, input_shape=(3, 32, 32)))
638 >>> model.output_shape
639 (None, 1, 10, 64)
641 >>> model.add(Flatten())
642 >>> model.output_shape
643 (None, 640)
645 """
647 def __init__(self, data_format=None, **kwargs):
648 super(Flatten, self).__init__(**kwargs)
649 self.data_format = conv_utils.normalize_data_format(data_format)
650 self.input_spec = InputSpec(min_ndim=1)
651 self._channels_first = self.data_format == 'channels_first'
653 def call(self, inputs):
654 if self._channels_first:
655 rank = inputs.shape.rank
656 if rank and rank > 1:
657 # Switch to channels-last format.
658 permutation = [0]
659 permutation.extend(range(2, rank))
660 permutation.append(1)
661 inputs = array_ops.transpose(inputs, perm=permutation)
663 if context.executing_eagerly():
664 # Full static shape is guaranteed to be available.
665 # Performance: Using `constant_op` is much faster than passing a list.
666 flattened_shape = constant_op.constant([inputs.shape[0], -1])
667 return array_ops.reshape(inputs, flattened_shape)
668 else:
669 input_shape = inputs.shape
670 rank = input_shape.rank
671 if rank == 1:
672 return array_ops.expand_dims_v2(inputs, axis=1)
673 else:
674 batch_dim = tensor_shape.dimension_value(input_shape[0])
675 non_batch_dims = input_shape[1:]
676 # Reshape in a way that preserves as much shape info as possible.
677 if non_batch_dims.is_fully_defined():
678 last_dim = int(functools.reduce(operator.mul, non_batch_dims))
679 flattened_shape = constant_op.constant([-1, last_dim])
680 elif batch_dim is not None:
681 flattened_shape = constant_op.constant([int(batch_dim), -1])
682 else:
683 flattened_shape = [array_ops.shape_v2(inputs)[0], -1]
684 return array_ops.reshape(inputs, flattened_shape)
686 def compute_output_shape(self, input_shape):
687 input_shape = tensor_shape.TensorShape(input_shape).as_list()
688 if not input_shape:
689 output_shape = tensor_shape.TensorShape([1])
690 else:
691 output_shape = [input_shape[0]]
692 if np.all(input_shape[1:]):
693 output_shape += [np.prod(input_shape[1:], dtype=int)]
694 else:
695 output_shape += [None]
696 return tensor_shape.TensorShape(output_shape)
698 def get_config(self):
699 config = super(Flatten, self).get_config()
700 config.update({'data_format': self.data_format})
701 return config
704@keras_export('keras.layers.RepeatVector')
705class RepeatVector(Layer):
706 """Repeats the input n times.
708 Example:
710 ```python
711 model = Sequential()
712 model.add(Dense(32, input_dim=32))
713 # now: model.output_shape == (None, 32)
714 # note: `None` is the batch dimension
716 model.add(RepeatVector(3))
717 # now: model.output_shape == (None, 3, 32)
718 ```
720 Args:
721 n: Integer, repetition factor.
723 Input shape:
724 2D tensor of shape `(num_samples, features)`.
726 Output shape:
727 3D tensor of shape `(num_samples, n, features)`.
728 """
730 def __init__(self, n, **kwargs):
731 super(RepeatVector, self).__init__(**kwargs)
732 self.n = n
733 if not isinstance(n, int):
734 raise TypeError(f'Expected an integer value for `n`, got {type(n)}.')
735 self.input_spec = InputSpec(ndim=2)
737 def compute_output_shape(self, input_shape):
738 input_shape = tensor_shape.TensorShape(input_shape).as_list()
739 return tensor_shape.TensorShape([input_shape[0], self.n, input_shape[1]])
741 def call(self, inputs):
742 return K.repeat(inputs, self.n)
744 def get_config(self):
745 config = {'n': self.n}
746 base_config = super(RepeatVector, self).get_config()
747 return dict(list(base_config.items()) + list(config.items()))
750@keras_export('keras.layers.Lambda')
751class Lambda(Layer):
752 """Wraps arbitrary expressions as a `Layer` object.
754 The `Lambda` layer exists so that arbitrary expressions can be used
755 as a `Layer` when constructing `Sequential`
756 and Functional API models. `Lambda` layers are best suited for simple
757 operations or quick experimentation. For more advanced use cases, follow
758 [this guide](https://www.tensorflow.org/guide/keras/custom_layers_and_models)
759 for subclassing `tf.keras.layers.Layer`.
761 WARNING: `tf.keras.layers.Lambda` layers have (de)serialization limitations!
763 The main reason to subclass `tf.keras.layers.Layer` instead of using a
764 `Lambda` layer is saving and inspecting a Model. `Lambda` layers
765 are saved by serializing the Python bytecode, which is fundamentally
766 non-portable. They should only be loaded in the same environment where
767 they were saved. Subclassed layers can be saved in a more portable way
768 by overriding their `get_config` method. Models that rely on
769 subclassed Layers are also often easier to visualize and reason about.
771 Examples:
773 ```python
774 # add a x -> x^2 layer
775 model.add(Lambda(lambda x: x ** 2))
776 ```
777 ```python
778 # add a layer that returns the concatenation
779 # of the positive part of the input and
780 # the opposite of the negative part
782 def antirectifier(x):
783 x -= K.mean(x, axis=1, keepdims=True)
784 x = K.l2_normalize(x, axis=1)
785 pos = K.relu(x)
786 neg = K.relu(-x)
787 return K.concatenate([pos, neg], axis=1)
789 model.add(Lambda(antirectifier))
790 ```
792 Variables:
793 While it is possible to use Variables with Lambda layers, this practice is
794 discouraged as it can easily lead to bugs. For instance, consider the
795 following layer:
797 ```python
798 scale = tf.Variable(1.)
799 scale_layer = tf.keras.layers.Lambda(lambda x: x * scale)
800 ```
802 Because scale_layer does not directly track the `scale` variable, it will
803 not appear in `scale_layer.trainable_weights` and will therefore not be
804 trained if `scale_layer` is used in a Model.
806 A better pattern is to write a subclassed Layer:
808 ```python
809 class ScaleLayer(tf.keras.layers.Layer):
810 def __init__(self):
811 super(ScaleLayer, self).__init__()
812 self.scale = tf.Variable(1.)
814 def call(self, inputs):
815 return inputs * self.scale
816 ```
818 In general, Lambda layers can be convenient for simple stateless
819 computation, but anything more complex should use a subclass Layer instead.
821 Args:
822 function: The function to be evaluated. Takes input tensor as first
823 argument.
824 output_shape: Expected output shape from function. This argument can be
825 inferred if not explicitly provided. Can be a tuple or function. If a
826 tuple, it only specifies the first dimension onward;
827 sample dimension is assumed either the same as the input: `output_shape =
828 (input_shape[0], ) + output_shape` or, the input is `None` and
829 the sample dimension is also `None`: `output_shape = (None, ) +
830 output_shape` If a function, it specifies the entire shape as a function
831 of the
832 input shape: `output_shape = f(input_shape)`
833 mask: Either None (indicating no masking) or a callable with the same
834 signature as the `compute_mask` layer method, or a tensor that will be
835 returned as output mask regardless of what the input is.
836 arguments: Optional dictionary of keyword arguments to be passed to the
837 function.
839 Input shape:
840 Arbitrary. Use the keyword argument input_shape (tuple of
841 integers, does not include the samples axis) when using this layer as the
842 first layer in a model.
844 Output shape:
845 Specified by `output_shape` argument
846 """
848 @trackable.no_automatic_dependency_tracking
849 def __init__(self, function, output_shape=None, mask=None, arguments=None,
850 **kwargs):
851 super(Lambda, self).__init__(**kwargs)
853 self.arguments = arguments or {}
854 self.function = function
856 if mask is not None:
857 self.supports_masking = True
858 self.mask = mask
859 self._output_shape = output_shape
861 # Warning on every invocation will be quite irksome in Eager mode.
862 self._already_warned = False
864 function_args = tf_inspect.getfullargspec(function).args
865 self._fn_expects_training_arg = 'training' in function_args
866 self._fn_expects_mask_arg = 'mask' in function_args
868 @tf_utils.shape_type_conversion
869 def compute_output_shape(self, input_shape):
870 if self._output_shape is None:
871 # Make use of existing autocomputation but provide Lambda-specific
872 # error message. This is always safe to run even when the outer context
873 # is Graph mode because Lambda layers don't have side effects such as
874 # `add_loss`.
875 with context.eager_mode():
876 try:
877 return super(Lambda, self).compute_output_shape(input_shape)
878 except NotImplementedError:
879 raise NotImplementedError(
880 'We could not automatically infer the shape of the Lambda\'s '
881 'output. Please specify `output_shape` for this Lambda.')
883 if callable(self._output_shape):
884 output_shapes = self._output_shape(input_shape)
885 return tf_utils.convert_shapes(output_shapes, to_tuples=False)
887 # Output shapes are passed directly and don't include batch dimension.
888 input_tensor_shape = tf_utils.convert_shapes(input_shape, to_tuples=False)
889 batch_size = nest.flatten(input_tensor_shape)[0][0] if input_shape else None
891 def _add_batch(shape):
892 return tensor_shape.TensorShape([batch_size] + shape.as_list())
894 output_shapes = tf_utils.convert_shapes(self._output_shape, to_tuples=False)
895 return nest.map_structure(_add_batch, output_shapes)
897 def call(self, inputs, mask=None, training=None):
898 # We must copy for thread safety, but it only needs to be a shallow copy.
899 kwargs = {k: v for k, v in self.arguments.items()}
900 if self._fn_expects_mask_arg:
901 kwargs['mask'] = mask
902 if self._fn_expects_training_arg:
903 kwargs['training'] = training
905 created_variables = []
906 def _variable_creator(next_creator, **kwargs):
907 var = next_creator(**kwargs)
908 created_variables.append(var)
909 return var
911 with backprop.GradientTape(watch_accessed_variables=True) as tape,\
912 variable_scope.variable_creator_scope(_variable_creator):
913 result = self.function(inputs, **kwargs)
914 self._check_variables(created_variables, tape.watched_variables())
915 return result
917 def _check_variables(self, created_variables, accessed_variables):
918 if not created_variables and not accessed_variables:
919 # In the common case that a Lambda layer does not touch a Variable, we
920 # don't want to incur the runtime cost of assembling any state used for
921 # checking only to immediately discard it.
922 return
924 tracked_weights = set(v.ref() for v in self.weights)
925 untracked_new_vars = [
926 v for v in created_variables if v.ref() not in tracked_weights
927 ]
928 if untracked_new_vars:
929 variable_str = '\n'.join(' {}'.format(i) for i in untracked_new_vars)
930 error_str = textwrap.dedent(
931 '''
932 The following Variables were created within a Lambda layer ({name})
933 but are not tracked by said layer:
934 {variable_str}
935 The layer cannot safely ensure proper Variable reuse across multiple
936 calls, and consquently this behavior is disallowed for safety. Lambda
937 layers are not well suited to stateful computation; instead, writing a
938 subclassed Layer is the recommend way to define layers with
939 Variables.'''
940 ).format(name=self.name, variable_str=variable_str)
941 raise ValueError(error_str)
943 untracked_used_vars = [
944 v for v in accessed_variables if v.ref() not in tracked_weights
945 ]
946 if untracked_used_vars and not self._already_warned:
947 variable_str = '\n'.join(' {}'.format(i) for i in untracked_used_vars)
948 self._warn(textwrap.dedent(
949 '''
950 The following Variables were used a Lambda layer's call ({name}), but
951 are not present in its tracked objects:
952 {variable_str}
953 It is possible that this is intended behavior, but it is more likely
954 an omission. This is a strong indication that this layer should be
955 formulated as a subclassed Layer rather than a Lambda layer.'''
956 ).format(name=self.name, variable_str=variable_str))
957 self._already_warned = True
959 def _warn(self, msg):
960 # This method will be overridden in a unit test to raise an error, because
961 # self.assertWarns is not universally implemented.
962 return tf_logging.warning(msg)
964 def compute_mask(self, inputs, mask=None):
965 if callable(self.mask):
966 return self.mask(inputs, mask)
967 return self.mask
969 def get_config(self):
970 function_config = self._serialize_function_to_config(self.function)
971 output_shape_config = self._serialize_function_to_config(self._output_shape,
972 allow_raw=True)
973 config = {
974 'function': function_config[0],
975 'function_type': function_config[1],
976 'module': function_config[2],
977 'output_shape': output_shape_config[0],
978 'output_shape_type': output_shape_config[1],
979 'output_shape_module': output_shape_config[2],
980 }
981 if self.mask is not None:
982 mask_config = self._serialize_function_to_config(self.mask)
983 config.update({
984 'mask': mask_config[0],
985 'mask_type': mask_config[1],
986 'mask_module': mask_config[2]
987 })
988 config['arguments'] = self.arguments
990 base_config = super(Lambda, self).get_config()
991 return dict(list(base_config.items()) + list(config.items()))
993 def _serialize_function_to_config(self, inputs, allow_raw=False):
994 if isinstance(inputs, python_types.LambdaType):
995 output = generic_utils.func_dump(inputs)
996 output_type = 'lambda'
997 module = inputs.__module__
998 elif callable(inputs):
999 output = inputs.__name__
1000 output_type = 'function'
1001 module = inputs.__module__
1002 elif allow_raw:
1003 output = inputs
1004 output_type = 'raw'
1005 module = None
1006 else:
1007 raise ValueError(
1008 'Invalid input for serialization, type: %s ' % type(inputs))
1010 return output, output_type, module
1012 @classmethod
1013 def from_config(cls, config, custom_objects=None):
1014 config = config.copy()
1015 function = cls._parse_function_from_config(
1016 config, custom_objects, 'function', 'module', 'function_type')
1018 output_shape = cls._parse_function_from_config(
1019 config, custom_objects, 'output_shape', 'output_shape_module',
1020 'output_shape_type')
1021 if 'mask' in config:
1022 mask = cls._parse_function_from_config(
1023 config, custom_objects, 'mask', 'mask_module', 'mask_type')
1024 else:
1025 mask = None
1027 config['function'] = function
1028 config['output_shape'] = output_shape
1029 config['mask'] = mask
1031 # If arguments were numpy array, they have been saved as
1032 # list. We need to recover the ndarray
1033 if 'arguments' in config:
1034 for key in config['arguments']:
1035 if isinstance(config['arguments'][key], dict):
1036 arg_dict = config['arguments'][key]
1037 if 'type' in arg_dict and arg_dict['type'] == 'ndarray':
1038 # Overwrite the argument with its numpy translation
1039 config['arguments'][key] = np.array(arg_dict['value'])
1041 return cls(**config)
1043 @classmethod
1044 def _parse_function_from_config(
1045 cls, config, custom_objects, func_attr_name, module_attr_name,
1046 func_type_attr_name):
1047 globs = globals().copy()
1048 module = config.pop(module_attr_name, None)
1049 if module in sys.modules:
1050 globs.update(sys.modules[module].__dict__)
1051 elif module is not None:
1052 # Note: we don't know the name of the function if it's a lambda.
1053 warnings.warn('{} is not loaded, but a Lambda layer uses it. '
1054 'It may cause errors.'.format(module)
1055 , UserWarning)
1056 if custom_objects:
1057 globs.update(custom_objects)
1058 function_type = config.pop(func_type_attr_name)
1059 if function_type == 'function':
1060 # Simple lookup in custom objects
1061 function = generic_utils.deserialize_keras_object(
1062 config[func_attr_name],
1063 custom_objects=custom_objects,
1064 printable_module_name='function in Lambda layer')
1065 elif function_type == 'lambda':
1066 # Unsafe deserialization from bytecode
1067 function = generic_utils.func_load(
1068 config[func_attr_name], globs=globs)
1069 elif function_type == 'raw':
1070 function = config[func_attr_name]
1071 else:
1072 raise TypeError('Unknown function type:', function_type)
1073 return function
1076@keras_export('keras.layers.Dense')
1077class Dense(Layer):
1078 """Just your regular densely-connected NN layer.
1080 `Dense` implements the operation:
1081 `output = activation(dot(input, kernel) + bias)`
1082 where `activation` is the element-wise activation function
1083 passed as the `activation` argument, `kernel` is a weights matrix
1084 created by the layer, and `bias` is a bias vector created by the layer
1085 (only applicable if `use_bias` is `True`). These are all attributes of
1086 `Dense`.
1088 Note: If the input to the layer has a rank greater than 2, then `Dense`
1089 computes the dot product between the `inputs` and the `kernel` along the
1090 last axis of the `inputs` and axis 0 of the `kernel` (using `tf.tensordot`).
1091 For example, if input has dimensions `(batch_size, d0, d1)`,
1092 then we create a `kernel` with shape `(d1, units)`, and the `kernel` operates
1093 along axis 2 of the `input`, on every sub-tensor of shape `(1, 1, d1)`
1094 (there are `batch_size * d0` such sub-tensors).
1095 The output in this case will have shape `(batch_size, d0, units)`.
1097 Besides, layer attributes cannot be modified after the layer has been called
1098 once (except the `trainable` attribute).
1099 When a popular kwarg `input_shape` is passed, then keras will create
1100 an input layer to insert before the current layer. This can be treated
1101 equivalent to explicitly defining an `InputLayer`.
1103 Example:
1105 >>> # Create a `Sequential` model and add a Dense layer as the first layer.
1106 >>> model = tf.keras.models.Sequential()
1107 >>> model.add(tf.keras.Input(shape=(16,)))
1108 >>> model.add(tf.keras.layers.Dense(32, activation='relu'))
1109 >>> # Now the model will take as input arrays of shape (None, 16)
1110 >>> # and output arrays of shape (None, 32).
1111 >>> # Note that after the first layer, you don't need to specify
1112 >>> # the size of the input anymore:
1113 >>> model.add(tf.keras.layers.Dense(32))
1114 >>> model.output_shape
1115 (None, 32)
1117 Args:
1118 units: Positive integer, dimensionality of the output space.
1119 activation: Activation function to use.
1120 If you don't specify anything, no activation is applied
1121 (ie. "linear" activation: `a(x) = x`).
1122 use_bias: Boolean, whether the layer uses a bias vector.
1123 kernel_initializer: Initializer for the `kernel` weights matrix.
1124 bias_initializer: Initializer for the bias vector.
1125 kernel_regularizer: Regularizer function applied to
1126 the `kernel` weights matrix.
1127 bias_regularizer: Regularizer function applied to the bias vector.
1128 activity_regularizer: Regularizer function applied to
1129 the output of the layer (its "activation").
1130 kernel_constraint: Constraint function applied to
1131 the `kernel` weights matrix.
1132 bias_constraint: Constraint function applied to the bias vector.
1134 Input shape:
1135 N-D tensor with shape: `(batch_size, ..., input_dim)`.
1136 The most common situation would be
1137 a 2D input with shape `(batch_size, input_dim)`.
1139 Output shape:
1140 N-D tensor with shape: `(batch_size, ..., units)`.
1141 For instance, for a 2D input with shape `(batch_size, input_dim)`,
1142 the output would have shape `(batch_size, units)`.
1143 """
1145 def __init__(self,
1146 units,
1147 activation=None,
1148 use_bias=True,
1149 kernel_initializer='glorot_uniform',
1150 bias_initializer='zeros',
1151 kernel_regularizer=None,
1152 bias_regularizer=None,
1153 activity_regularizer=None,
1154 kernel_constraint=None,
1155 bias_constraint=None,
1156 **kwargs):
1157 super(Dense, self).__init__(
1158 activity_regularizer=activity_regularizer, **kwargs)
1160 self.units = int(units) if not isinstance(units, int) else units
1161 if self.units < 0:
1162 raise ValueError(f'Received an invalid value for `units`, expected '
1163 f'a positive integer, got {units}.')
1164 self.activation = activations.get(activation)
1165 self.use_bias = use_bias
1166 self.kernel_initializer = initializers.get(kernel_initializer)
1167 self.bias_initializer = initializers.get(bias_initializer)
1168 self.kernel_regularizer = regularizers.get(kernel_regularizer)
1169 self.bias_regularizer = regularizers.get(bias_regularizer)
1170 self.kernel_constraint = constraints.get(kernel_constraint)
1171 self.bias_constraint = constraints.get(bias_constraint)
1173 self.input_spec = InputSpec(min_ndim=2)
1174 self.supports_masking = True
1176 def build(self, input_shape):
1177 dtype = dtypes.as_dtype(self.dtype or K.floatx())
1178 if not (dtype.is_floating or dtype.is_complex):
1179 raise TypeError('Unable to build `Dense` layer with non-floating point '
1180 'dtype %s' % (dtype,))
1182 input_shape = tensor_shape.TensorShape(input_shape)
1183 last_dim = tensor_shape.dimension_value(input_shape[-1])
1184 if last_dim is None:
1185 raise ValueError('The last dimension of the inputs to `Dense` '
1186 'should be defined. Found `None`.')
1187 self.input_spec = InputSpec(min_ndim=2, axes={-1: last_dim})
1188 self.kernel = self.add_weight(
1189 'kernel',
1190 shape=[last_dim, self.units],
1191 initializer=self.kernel_initializer,
1192 regularizer=self.kernel_regularizer,
1193 constraint=self.kernel_constraint,
1194 dtype=self.dtype,
1195 trainable=True)
1196 if self.use_bias:
1197 self.bias = self.add_weight(
1198 'bias',
1199 shape=[self.units,],
1200 initializer=self.bias_initializer,
1201 regularizer=self.bias_regularizer,
1202 constraint=self.bias_constraint,
1203 dtype=self.dtype,
1204 trainable=True)
1205 else:
1206 self.bias = None
1207 self.built = True
1209 def call(self, inputs):
1210 if inputs.dtype.base_dtype != self._compute_dtype_object.base_dtype:
1211 inputs = math_ops.cast(inputs, dtype=self._compute_dtype_object)
1213 rank = inputs.shape.rank
1214 if rank == 2 or rank is None:
1215 # We use embedding_lookup_sparse as a more efficient matmul operation for
1216 # large sparse input tensors. The op will result in a sparse gradient, as
1217 # opposed to sparse_ops.sparse_tensor_dense_matmul which results in dense
1218 # gradients. This can lead to sigfinicant speedups, see b/171762937.
1219 if isinstance(inputs, sparse_tensor.SparseTensor):
1220 # We need to fill empty rows, as the op assumes at least one id per row.
1221 inputs, _ = sparse_ops.sparse_fill_empty_rows(inputs, 0)
1222 # We need to do some munging of our input to use the embedding lookup as
1223 # a matrix multiply. We split our input matrix into separate ids and
1224 # weights tensors. The values of the ids tensor should be the column
1225 # indices of our input matrix and the values of the weights tensor
1226 # can continue to the actual matrix weights.
1227 # The column arrangement of ids and weights
1228 # will be summed over and does not matter. See the documentation for
1229 # sparse_ops.sparse_tensor_dense_matmul a more detailed explanation
1230 # of the inputs to both ops.
1231 ids = sparse_tensor.SparseTensor(
1232 indices=inputs.indices,
1233 values=inputs.indices[:, 1],
1234 dense_shape=inputs.dense_shape)
1235 weights = inputs
1236 outputs = embedding_ops.embedding_lookup_sparse_v2(
1237 self.kernel, ids, weights, combiner='sum')
1238 else:
1239 outputs = gen_math_ops.MatMul(a=inputs, b=self.kernel)
1240 # Broadcast kernel to inputs.
1241 else:
1242 outputs = standard_ops.tensordot(inputs, self.kernel, [[rank - 1], [0]])
1243 # Reshape the output back to the original ndim of the input.
1244 if not context.executing_eagerly():
1245 shape = inputs.shape.as_list()
1246 output_shape = shape[:-1] + [self.kernel.shape[-1]]
1247 outputs.set_shape(output_shape)
1249 if self.use_bias:
1250 outputs = nn_ops.bias_add(outputs, self.bias)
1252 if self.activation is not None:
1253 outputs = self.activation(outputs)
1254 return outputs
1256 def compute_output_shape(self, input_shape):
1257 input_shape = tensor_shape.TensorShape(input_shape)
1258 input_shape = input_shape.with_rank_at_least(2)
1259 if tensor_shape.dimension_value(input_shape[-1]) is None:
1260 raise ValueError(
1261 'The innermost dimension of input_shape must be defined, but saw: %s'
1262 % (input_shape,))
1263 return input_shape[:-1].concatenate(self.units)
1265 def get_config(self):
1266 config = super(Dense, self).get_config()
1267 config.update({
1268 'units': self.units,
1269 'activation': activations.serialize(self.activation),
1270 'use_bias': self.use_bias,
1271 'kernel_initializer': initializers.serialize(self.kernel_initializer),
1272 'bias_initializer': initializers.serialize(self.bias_initializer),
1273 'kernel_regularizer': regularizers.serialize(self.kernel_regularizer),
1274 'bias_regularizer': regularizers.serialize(self.bias_regularizer),
1275 'activity_regularizer':
1276 regularizers.serialize(self.activity_regularizer),
1277 'kernel_constraint': constraints.serialize(self.kernel_constraint),
1278 'bias_constraint': constraints.serialize(self.bias_constraint)
1279 })
1280 return config
1283@keras_export('keras.layers.ActivityRegularization')
1284class ActivityRegularization(Layer):
1285 """Layer that applies an update to the cost function based input activity.
1287 Args:
1288 l1: L1 regularization factor (positive float).
1289 l2: L2 regularization factor (positive float).
1291 Input shape:
1292 Arbitrary. Use the keyword argument `input_shape`
1293 (tuple of integers, does not include the samples axis)
1294 when using this layer as the first layer in a model.
1296 Output shape:
1297 Same shape as input.
1298 """
1300 def __init__(self, l1=0., l2=0., **kwargs):
1301 super(ActivityRegularization, self).__init__(
1302 activity_regularizer=regularizers.L1L2(l1=l1, l2=l2), **kwargs)
1303 self.supports_masking = True
1304 self.l1 = l1
1305 self.l2 = l2
1307 def compute_output_shape(self, input_shape):
1308 return input_shape
1310 def get_config(self):
1311 config = {'l1': self.l1, 'l2': self.l2}
1312 base_config = super(ActivityRegularization, self).get_config()
1313 return dict(list(base_config.items()) + list(config.items()))
1316class TFOpLambda(Layer):
1317 """Wraps TF API symbols in a `Layer` object.
1319 It is inserted by the Functional API construction whenever users call
1320 a supported TF symbol on KerasTensors.
1322 Like Lambda layers, this layer tries to raise warnings when it detects users
1323 explicitly use variables in the call. (To let them know
1324 that the layer will not capture the variables).
1326 This is useful in the case where users do something like:
1327 x = keras.Input(...)
1328 y = tf.Variable(...)
1329 out = x * tf_variable
1330 """
1332 @trackable.no_automatic_dependency_tracking
1333 def __init__(self, function, **kwargs):
1334 self.function = function
1335 self.symbol = (
1336 get_canonical_name_for_symbol(self.function,
1337 add_prefix_to_v1_names=True) or
1338 get_canonical_name_for_symbol(self.function,
1339 api_name='keras',
1340 add_prefix_to_v1_names=True))
1341 if 'name' not in kwargs:
1342 # Generate a name.
1343 # TFOpLambda layers avoid already-observed names,
1344 # because users cannot easily control the generated names.
1345 # Without this avoidance, users would be more likely to run
1346 # into unavoidable duplicate layer name collisions.
1347 # (For standard layers users could just set `name` when creating the
1348 # layer to work around a collision, but they can't do that for
1349 # auto-generated layers)
1350 if self.symbol:
1351 name = 'tf.' + self.symbol
1352 else:
1353 name = self.function.__name__
1354 kwargs['name'] = K.unique_object_name(
1355 name, zero_based=True, avoid_observed_names=True)
1356 kwargs['autocast'] = False
1358 # Decorate the function to produce this layer's call method
1359 def _call_wrapper(*args, **kwargs):
1360 return self._call_wrapper(*args, **kwargs)
1361 self.call = tf_decorator.make_decorator(function, _call_wrapper)
1363 # Do not individually trace op layers in the SavedModel.
1364 self._must_restore_from_config = True
1366 super(TFOpLambda, self).__init__(**kwargs)
1368 # Preserve all argument data structures when saving/loading a config
1369 # (e.g., don't unnest lists that contain one element)
1370 self._preserve_input_structure_in_config = True
1372 # Warning on every invocation will be quite irksome in Eager mode.
1373 self._already_warned = False
1375 self._expects_training_arg = False
1376 self._expects_mask_arg = False
1378 def _call_wrapper(self, *args, **kwargs):
1379 created_variables = []
1380 def _variable_creator(next_creator, **creator_kwargs):
1381 var = next_creator(**creator_kwargs)
1382 created_variables.append(var)
1383 return var
1385 with backprop.GradientTape(watch_accessed_variables=True) as tape, \
1386 variable_scope.variable_creator_scope(_variable_creator):
1387 # We explicitly drop `name` arguments here,
1388 # to guard against the case where an op explicitly has a
1389 # `name` passed (which is susceptible to producing
1390 # multiple ops w/ the same name when the layer is reused)
1391 kwargs.pop('name', None)
1392 result = self.function(*args, **kwargs)
1393 self._check_variables(created_variables, tape.watched_variables())
1394 return result
1396 def _check_variables(self, created_variables, accessed_variables):
1397 if not created_variables and not accessed_variables:
1398 # In the common case that a Lambda layer does not touch a Variable, we
1399 # don't want to incur the runtime cost of assembling any state used for
1400 # checking only to immediately discard it.
1401 return
1403 tracked_weights = set(v.ref() for v in self.weights)
1404 untracked_new_vars = [
1405 v for v in created_variables if v.ref() not in tracked_weights
1406 ]
1407 if untracked_new_vars:
1408 variable_str = '\n'.join(' {}'.format(i) for i in untracked_new_vars)
1409 error_str = textwrap.dedent(
1410 '''
1411 The following Variables were created within a Lambda layer ({name})
1412 but are not tracked by said layer:
1413 {variable_str}
1414 The layer cannot safely ensure proper Variable reuse across multiple
1415 calls, and consquently this behavior is disallowed for safety. Lambda
1416 layers are not well suited to stateful computation; instead, writing a
1417 subclassed Layer is the recommend way to define layers with
1418 Variables.'''
1419 ).format(name=self.name, variable_str=variable_str)
1420 raise ValueError(error_str)
1422 untracked_used_vars = [
1423 v for v in accessed_variables if v.ref() not in tracked_weights
1424 ]
1425 if untracked_used_vars and not self._already_warned:
1426 variable_str = '\n'.join(' {}'.format(i) for i in untracked_used_vars)
1427 self._warn(textwrap.dedent(
1428 '''
1429 The following Variables were used a Lambda layer's call ({name}), but
1430 are not present in its tracked objects:
1431 {variable_str}
1432 It is possible that this is intended behavior, but it is more likely
1433 an omission. This is a strong indication that this layer should be
1434 formulated as a subclassed Layer rather than a Lambda layer.'''
1435 ).format(name=self.name, variable_str=variable_str))
1436 self._already_warned = True
1438 def _warn(self, msg):
1439 # This method will be overridden in a unit test to raise an error, because
1440 # self.assertWarns is not universally implemented.
1441 return tf_logging.warning(msg)
1443 def get_config(self):
1444 if not self.symbol:
1445 raise ValueError('This Keras op layer was generated from %s, a method '
1446 'that is not an exposed in the TensorFlow API. This '
1447 'may have happened if the method was explicitly '
1448 'decorated to add dispatching support, and it was used '
1449 'during Functional model construction. '
1450 'To ensure cross-version compatibility of Keras models '
1451 'that use op layers, only op layers produced from '
1452 'exported TF API symbols can be serialized.'
1453 % self.function)
1454 config = {
1455 'function': self.symbol
1456 }
1458 base_config = super(TFOpLambda, self).get_config()
1459 return dict(list(base_config.items()) + list(config.items()))
1461 @classmethod
1462 def from_config(cls, config, custom_objects=None):
1463 config = config.copy()
1464 symbol_name = config['function']
1465 function = get_symbol_from_name(symbol_name)
1466 if not function:
1467 raise ValueError(
1468 'TF symbol `tf.%s` could not be found.' % symbol_name)
1470 config['function'] = function
1472 return cls(**config)
1475class KerasOpDispatcher(dispatch.GlobalOpDispatcher):
1476 """A global dispatcher that allows building a functional model with TF Ops."""
1478 def handle(self, op, args, kwargs):
1479 """Handle the specified operation with the specified arguments."""
1480 if any(
1481 isinstance(x, keras_tensor.KerasTensor)
1482 for x in nest.flatten([args, kwargs])):
1483 return TFOpLambda(op)(*args, **kwargs)
1484 else:
1485 return self.NOT_SUPPORTED
1487KerasOpDispatcher().register()
1490def _slice_to_dict(x):
1491 if isinstance(x, slice):
1492 return {'start': x.start, 'stop': x.stop, 'step': x.step}
1493 return x
1496def _dict_to_slice(x):
1497 if isinstance(x, dict):
1498 return slice(x['start'], x['stop'], x['step'])
1499 return x
1502class SlicingOpLambda(TFOpLambda):
1503 """Wraps TF API symbols in a `Layer` object.
1505 It is inserted by the Functional API construction whenever users call
1506 a supported TF symbol on KerasTensors.
1508 Like Lambda layers, this layer tries to raise warnings when it detects users
1509 explicitly use variables in the call. (To let them know
1510 that the layer will not capture the variables).
1512 This is useful in the case where users do something like:
1513 x = keras.Input(...)
1514 y = tf.Variable(...)
1515 out = x * tf_variable
1516 """
1518 @trackable.no_automatic_dependency_tracking
1519 def __init__(self, function, **kwargs):
1520 super(SlicingOpLambda, self).__init__(function, **kwargs)
1522 original_call = self.call
1523 # Decorate the function to produce this layer's call method
1524 def _call_wrapper(*args, **kwargs):
1525 # Turn any slice dicts in the args back into `slice` objects.
1526 # This conversion cannot use nest.flatten/map_structure,
1527 # because dicts are flattened by nest while slices aren't.
1528 # So, map_structure would only see the individual elements in the
1529 # dict.
1530 # This can't use map_structure_up_to either because the 'shallowness' of
1531 # the shallow tree would have to vary depending on if only one dim or
1532 # multiple are being sliced.
1533 new_args = []
1534 for arg in args:
1535 arg = _dict_to_slice(arg)
1536 if isinstance(arg, (list, tuple)):
1537 new_arg = []
1538 for sub_arg in arg:
1539 new_arg.append(_dict_to_slice(sub_arg))
1540 arg = new_arg
1541 new_args.append(arg)
1543 # Handle the kwargs too.
1544 new_kwargs = {}
1545 for key, value in kwargs.items():
1546 value = _dict_to_slice(value)
1547 if isinstance(value, (list, tuple)):
1548 new_value = []
1549 for v in value:
1550 new_value.append(_dict_to_slice(v))
1551 value = new_value
1552 new_kwargs[key] = value
1554 return original_call(*new_args, **new_kwargs)
1555 self.call = tf_decorator.make_decorator(original_call, _call_wrapper)
1558class TFSlicingOpDispatcher(dispatch.OpDispatcher):
1559 """A global dispatcher that allows building a functional model with TF Ops."""
1561 def __init__(self, op):
1562 self.op = op
1564 def handle(self, args, kwargs):
1565 """Handle the specified operation with the specified arguments."""
1566 args = nest.map_structure(_slice_to_dict, args)
1567 kwargs = nest.map_structure(_slice_to_dict, kwargs)
1568 if any(
1569 isinstance(x, keras_tensor.KerasTensor)
1570 for x in nest.flatten([args, kwargs])):
1571 return SlicingOpLambda(self.op)(*args, **kwargs)
1572 else:
1573 return self.NOT_SUPPORTED
1575for slicing_op in [
1576 array_ops._slice_helper, # pylint: disable=protected-access
1577 array_ops.boolean_mask,
1578 array_ops.boolean_mask_v2,
1579 ragged_getitem.ragged_tensor_getitem
1580]:
1581 TFSlicingOpDispatcher(slicing_op).register(slicing_op)
1584class InstanceProperty(Layer):
1585 """Wraps an instance property access (e.g. `x.foo`) in a Keras Layer.
1587 This layer takes an attribute name `attr_name` in the constructor and,
1588 when called on input tensor `obj` returns `obj.attr_name`.
1590 KerasTensors specialized for specific extension types use it to
1591 represent instance property accesses on the represented object in the
1592 case where the property needs to be dynamically accessed as opposed to
1593 being statically computed from the typespec, e.g.
1595 x = keras.Input(..., ragged=True)
1596 out = x.flat_values
1597 """
1599 @trackable.no_automatic_dependency_tracking
1600 def __init__(self, attr_name, **kwargs):
1601 self.attr_name = attr_name
1603 if 'name' not in kwargs:
1604 kwargs['name'] = K.unique_object_name(
1605 'input.' + self.attr_name, zero_based=True, avoid_observed_names=True)
1606 kwargs['autocast'] = False
1608 # Do not individually trace op layers in the SavedModel.
1609 self._must_restore_from_config = True
1611 super(InstanceProperty, self).__init__(**kwargs)
1613 # Preserve all argument data structures when saving/loading a config
1614 # (e.g., don't unnest lists that contain one element)
1615 self._preserve_input_structure_in_config = True
1617 def call(self, obj):
1618 return getattr(obj, self.attr_name)
1620 def get_config(self):
1621 config = {
1622 'attr_name': self.attr_name
1623 }
1624 base_config = super(InstanceProperty, self).get_config()
1625 return dict(list(base_config.items()) + list(config.items()))
1627 @classmethod
1628 def from_config(cls, config, custom_objects=None):
1629 return cls(**config)
1632class InstanceMethod(InstanceProperty):
1633 """Wraps an instance method access (e.g. `x.foo(arg)` in a Keras Layer.
1635 This layer takes an attribute name `attr_name` in the constructor and,
1636 when called on input tensor `obj` with additional arguments `args` and
1637 `kwargs` returns `obj.attr_name(*args, **kwargs)`.
1639 KerasTensors specialized for specific extension types use it to
1640 represent dynamic instance method calls on the represented object, e.g.
1642 x = keras.Input(..., ragged=True)
1643 new_values = keras.Input(...)
1644 out = x.with_values(new_values)
1645 """
1647 def call(self, obj, args, kwargs):
1648 method = getattr(obj, self.attr_name)
1649 return method(*args, **kwargs)
1652def _delegate_property(keras_tensor_cls, property_name): # pylint: disable=invalid-name
1653 """Register property on a KerasTensor class.
1655 Calling this multiple times with the same arguments should be a no-op.
1657 This method exposes a property on the KerasTensor class that will use an
1658 `InstanceProperty` layer to access the property on the represented
1659 intermediate values in the model.
1661 Args:
1662 keras_tensor_cls: The KerasTensor subclass that should expose the property.
1663 property_name: The name of the property to expose and delegate to the
1664 represented (Composite)Tensor.
1665 """
1666 # We use a lambda because we can't create a Keras layer at import time
1667 # due to dynamic layer class versioning.
1668 property_access = property(lambda self: InstanceProperty(property_name)(self)) # pylint: disable=unnecessary-lambda
1669 setattr(keras_tensor_cls, property_name, property_access)
1672def _delegate_method(keras_tensor_cls, method_name): # pylint: disable=invalid-name
1673 """Register method on a KerasTensor class.
1675 Calling this function times with the same arguments should be a no-op.
1677 This method exposes an instance method on the KerasTensor class that will use
1678 an `InstanceMethod` layer to run the desired method on the represented
1679 intermediate values in the model.
1681 Args:
1682 keras_tensor_cls: The KerasTensor subclass that should expose the property.
1683 method_name: The name of the method to expose and delegate to the
1684 represented (Composite)Tensor.
1685 """
1686 def delegate(self, *args, **kwargs):
1687 return InstanceMethod(method_name)(self, args, kwargs)
1688 setattr(keras_tensor_cls, method_name, delegate)
1690# We do not support the `uniform_row_length` property because it
1691# returns either `None` or an int tensor, and code that relies on it tends
1692# to check `is None` directly. Delegating it here would always return a
1693# `KerasTensor`, regardless of what can be statically inferred. This would
1694# never equal `None`, breaking code that expects it to be partially-static
1695# in unpredictable ways.
1696for ragged_property in [
1697 'values',
1698 'flat_values',
1699 'row_splits',
1700 'nested_row_splits'
1701]:
1702 _delegate_property(keras_tensor.RaggedKerasTensor, ragged_property)
1704for ragged_method_name in [
1705 'value_rowids',
1706 'nested_value_rowids',
1707 'nrows',
1708 'row_starts',
1709 'row_limits',
1710 'row_lengths',
1711 'nested_row_lengths',
1712 'bounding_shape',
1713 'with_values',
1714 'with_flat_values',
1715 'with_row_splits_dtype',
1716 'merge_dims',
1717 'to_tensor',
1718 'to_sparse',
1719]:
1720 _delegate_method(keras_tensor.RaggedKerasTensor, ragged_method_name)
1722for sparse_property in [
1723 'indices',
1724 'values',
1725]:
1726 _delegate_property(keras_tensor.SparseKerasTensor, sparse_property)
1728for sparse_method in [
1729 'with_values',
1730]:
1731 _delegate_method(keras_tensor.SparseKerasTensor, sparse_method)
1734class ClassMethod(Layer):
1735 """Wraps a TF API Class's class method in a `Layer` object.
1737 It is inserted by the Functional API construction whenever users call
1738 a supported TF Class's class method on KerasTensors.
1740 This is useful in the case where users do something like:
1741 x = keras.Input(...)
1742 y = keras.Input(...)
1743 out = tf.RaggedTensor.from_row_splits(x, y)
1744 """
1746 @trackable.no_automatic_dependency_tracking
1747 def __init__(self, cls_ref, method_name, **kwargs):
1748 self.cls_ref = cls_ref
1749 self.method_name = method_name
1750 self.cls_symbol = (
1751 get_canonical_name_for_symbol(self.cls_ref,
1752 add_prefix_to_v1_names=True) or
1753 get_canonical_name_for_symbol(self.cls_ref,
1754 api_name='keras',
1755 add_prefix_to_v1_names=True))
1756 if 'name' not in kwargs:
1757 kwargs['name'] = K.unique_object_name(
1758 'tf.' + self.cls_symbol + '.' + self.method_name, zero_based=True,
1759 avoid_observed_names=True)
1760 kwargs['autocast'] = False
1762 # Do not individually trace op layers in the SavedModel.
1763 self._must_restore_from_config = True
1765 super(ClassMethod, self).__init__(**kwargs)
1767 # Preserve all argument data structures when saving/loading a config
1768 # (e.g., don't unnest lists that contain one element)
1769 self._preserve_input_structure_in_config = True
1771 self._expects_training_arg = False
1772 self._expects_mask_arg = False
1774 def call(self, args, kwargs):
1775 return getattr(self.cls_ref, self.method_name)(*args, **kwargs)
1777 def get_config(self):
1778 if not self.cls_symbol:
1779 raise ValueError('This Keras class method conversion tried to convert '
1780 'a method belonging to class %s, a class '
1781 'that is not an exposed in the TensorFlow API. '
1782 'To ensure cross-version compatibility of Keras models '
1783 'that use op layers, only op layers produced from '
1784 'exported TF API symbols can be serialized.'
1785 % self.cls_symbol)
1786 config = {
1787 'cls_symbol': self.cls_symbol,
1788 'method_name': self.method_name
1789 }
1791 base_config = super(ClassMethod, self).get_config()
1792 return dict(list(base_config.items()) + list(config.items()))
1794 @classmethod
1795 def from_config(cls, config, custom_objects=None):
1796 config = config.copy()
1797 symbol_name = config.pop('cls_symbol')
1798 cls_ref = get_symbol_from_name(symbol_name)
1799 if not cls_ref:
1800 raise ValueError(
1801 'TF symbol `tf.%s` could not be found.' % symbol_name)
1803 config['cls_ref'] = cls_ref
1805 return cls(**config)
1808class TFClassMethodDispatcher(dispatch.OpDispatcher):
1809 """A class method dispatcher that allows building a functional model with TF class methods."""
1811 def __init__(self, cls, method_name):
1812 self.cls = cls
1813 self.method_name = method_name
1815 def handle(self, args, kwargs):
1816 """Handle the specified operation with the specified arguments."""
1817 if any(
1818 isinstance(x, keras_tensor.KerasTensor)
1819 for x in nest.flatten([args, kwargs])):
1820 return ClassMethod(self.cls, self.method_name)(args[1:], kwargs)
1821 else:
1822 return self.NOT_SUPPORTED
1824for ragged_class_method in [
1825 'from_value_rowids',
1826 'from_row_splits',
1827 'from_row_lengths',
1828 'from_row_starts',
1829 'from_row_limits',
1830 'from_uniform_row_length',
1831 'from_nested_value_rowids',
1832 'from_nested_row_splits',
1833 'from_nested_row_lengths',
1834 'from_tensor',
1835 'from_sparse',
1836]:
1837 TFClassMethodDispatcher(
1838 ragged_tensor.RaggedTensor, ragged_class_method).register(
1839 getattr(ragged_tensor.RaggedTensor, ragged_class_method))