Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/tensorflow/python/keras/engine/base_layer.py: 25%

1# Copyright 2015 The TensorFlow Authors. All Rights Reserved. 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================== 

15# pylint: disable=protected-access 

16"""Contains the base Layer class, from which all layers inherit.""" 

17 

18import collections 

19import copy 

20import functools 

21import itertools 

22import threading 

23import warnings 

24import weakref 

25 

26import numpy as np 

27 

28from google.protobuf import json_format 

29from tensorflow.core.framework import node_def_pb2 

30from tensorflow.python import tf2 

31from tensorflow.python.autograph.core import ag_ctx 

32from tensorflow.python.autograph.impl import api as autograph 

33from tensorflow.python.distribute import distribute_lib 

34from tensorflow.python.eager import backprop 

35from tensorflow.python.eager import context 

36from tensorflow.python.eager import def_function 

37from tensorflow.python.framework import constant_op 

38from tensorflow.python.framework import dtypes 

39from tensorflow.python.framework import func_graph 

40from tensorflow.python.framework import ops 

41from tensorflow.python.framework import sparse_tensor 

42from tensorflow.python.framework import tensor_conversion 

43from tensorflow.python.framework import tensor_spec 

44from tensorflow.python.framework import tensor_util 

45from tensorflow.python.keras import backend 

46from tensorflow.python.keras import constraints 

47from tensorflow.python.keras import initializers 

48from tensorflow.python.keras import regularizers 

49from tensorflow.python.keras.engine import base_layer_utils 

50from tensorflow.python.keras.engine import input_spec 

51from tensorflow.python.keras.engine import keras_tensor 

52from tensorflow.python.keras.engine import node as node_module 

53from tensorflow.python.keras.mixed_precision import autocast_variable 

54from tensorflow.python.keras.mixed_precision import loss_scale_optimizer 

55from tensorflow.python.keras.mixed_precision import policy 

56from tensorflow.python.keras.saving.saved_model import layer_serialization 

57from tensorflow.python.keras.utils import generic_utils 

58from tensorflow.python.keras.utils import layer_utils 

59from tensorflow.python.keras.utils import object_identity 

60from tensorflow.python.keras.utils import tf_inspect 

61from tensorflow.python.keras.utils import tf_utils 

62from tensorflow.python.keras.utils import version_utils 

63from tensorflow.python.keras.utils.generic_utils import to_snake_case # pylint: disable=unused-import 

64from tensorflow.python.keras.utils.tf_utils import is_tensor_or_tensor_list # pylint: disable=unused-import 

65from tensorflow.python.module import module 

66from tensorflow.python.ops import array_ops 

67from tensorflow.python.ops import math_ops 

68from tensorflow.python.ops import variables as tf_variables 

69from tensorflow.python.ops.numpy_ops import np_arrays 

70from tensorflow.python.ops.ragged import ragged_tensor 

71from tensorflow.python.platform import tf_logging 

72from tensorflow.python.trackable import autotrackable 

73from tensorflow.python.trackable import base as trackable 

74from tensorflow.python.trackable import data_structures 

75from tensorflow.python.util import compat 

76from tensorflow.python.util import nest 

77from tensorflow.python.util.tf_export import get_canonical_name_for_symbol 

78from tensorflow.python.util.tf_export import keras_export 

79from tensorflow.tools.docs import doc_controls 

80 

81# A module that only depends on `keras.layers` import these from here. 

82 

83# pylint: disable=g-inconsistent-quotes 

84metrics_mod = generic_utils.LazyLoader( 

85 "metrics_mod", globals(), 

86 "tensorflow.python.keras.metrics") 

87# pylint: enable=g-inconsistent-quotes 

88 

89# Prefix that is added to the TF op layer names. 

90_TF_OP_LAYER_NAME_PREFIX = 'tf_op_layer_' 

91 

92# TODO(mdan): Should we have a single generic type for types that can be passed 

93# to tf.cast? 

94_AUTOCAST_TYPES = (ops.Tensor, sparse_tensor.SparseTensor, 

95 ragged_tensor.RaggedTensor) 

96 

97 

98@keras_export('keras.layers.Layer') 

99class Layer(module.Module, version_utils.LayerVersionSelector): 

100 """This is the class from which all layers inherit. 

101 

102 A layer is a callable object that takes as input one or more tensors and 

103 that outputs one or more tensors. It involves *computation*, defined 

104 in the `call()` method, and a *state* (weight variables), defined 

105 either in the constructor `__init__()` or in the `build()` method. 

106 

107 Users will just instantiate a layer and then treat it as a callable. 

108 

109 Args: 

110 trainable: Boolean, whether the layer's variables should be trainable. 

111 name: String name of the layer. 

112 dtype: The dtype of the layer's computations and weights. Can also be a 

113 `tf.keras.mixed_precision.Policy`, which allows the computation and weight 

114 dtype to differ. Default of `None` means to use 

115 `tf.keras.mixed_precision.global_policy()`, which is a float32 policy 

116 unless set to different value. 

117 dynamic: Set this to `True` if your layer should only be run eagerly, and 

118 should not be used to generate a static computation graph. 

119 This would be the case for a Tree-RNN or a recursive network, 

120 for example, or generally for any layer that manipulates tensors 

121 using Python control flow. If `False`, we assume that the layer can 

122 safely be used to generate a static computation graph. 

123 

124 Attributes: 

125 name: The name of the layer (string). 

126 dtype: The dtype of the layer's weights. 

127 variable_dtype: Alias of `dtype`. 

128 compute_dtype: The dtype of the layer's computations. Layers automatically 

129 cast inputs to this dtype which causes the computations and output to also 

130 be in this dtype. When mixed precision is used with a 

131 `tf.keras.mixed_precision.Policy`, this will be different than 

132 `variable_dtype`. 

133 dtype_policy: The layer's dtype policy. See the 

134 `tf.keras.mixed_precision.Policy` documentation for details. 

135 trainable_weights: List of variables to be included in backprop. 

136 non_trainable_weights: List of variables that should not be 

137 included in backprop. 

138 weights: The concatenation of the lists trainable_weights and 

139 non_trainable_weights (in this order). 

140 trainable: Whether the layer should be trained (boolean), i.e. whether 

141 its potentially-trainable weights should be returned as part of 

142 `layer.trainable_weights`. 

143 input_spec: Optional (list of) `InputSpec` object(s) specifying the 

144 constraints on inputs that can be accepted by the layer. 

145 

146 We recommend that descendants of `Layer` implement the following methods: 

147 

148 * `__init__()`: Defines custom layer attributes, and creates layer state 

149 variables that do not depend on input shapes, using `add_weight()`. 

150 * `build(self, input_shape)`: This method can be used to create weights that 

151 depend on the shape(s) of the input(s), using `add_weight()`. `__call__()` 

152 will automatically build the layer (if it has not been built yet) by 

153 calling `build()`. 

154 * `call(self, inputs, *args, **kwargs)`: Called in `__call__` after making 

155 sure `build()` has been called. `call()` performs the logic of applying the 

156 layer to the input tensors (which should be passed in as argument). 

157 Two reserved keyword arguments you can optionally use in `call()` are: 

158 - `training` (boolean, whether the call is in inference mode or training 

159 mode). See more details in [the layer/model subclassing guide]( 

160 https://www.tensorflow.org/guide/keras/custom_layers_and_models#privileged_training_argument_in_the_call_method) 

161 - `mask` (boolean tensor encoding masked timesteps in the input, used 

162 in RNN layers). See more details in [the layer/model subclassing guide]( 

163 https://www.tensorflow.org/guide/keras/custom_layers_and_models#privileged_mask_argument_in_the_call_method) 

164 A typical signature for this method is `call(self, inputs)`, and user could 

165 optionally add `training` and `mask` if the layer need them. `*args` and 

166 `**kwargs` is only useful for future extension when more input parameters 

167 are planned to be added. 

168 * `get_config(self)`: Returns a dictionary containing the configuration used 

169 to initialize this layer. If the keys differ from the arguments 

170 in `__init__`, then override `from_config(self)` as well. 

171 This method is used when saving 

172 the layer or a model that contains this layer. 

173 

174 Examples: 

175 

176 Here's a basic example: a layer with two variables, `w` and `b`, 

177 that returns `y = w . x + b`. 

178 It shows how to implement `build()` and `call()`. 

179 Variables set as attributes of a layer are tracked as weights 

180 of the layers (in `layer.weights`). 

181 

182 ```python 

183 class SimpleDense(Layer): 

184 

185 def __init__(self, units=32): 

186 super(SimpleDense, self).__init__() 

187 self.units = units 

188 

189 def build(self, input_shape): # Create the state of the layer (weights) 

190 w_init = tf.random_normal_initializer() 

191 self.w = tf.Variable( 

192 initial_value=w_init(shape=(input_shape[-1], self.units), 

193 dtype='float32'), 

194 trainable=True) 

195 b_init = tf.zeros_initializer() 

196 self.b = tf.Variable( 

197 initial_value=b_init(shape=(self.units,), dtype='float32'), 

198 trainable=True) 

199 

200 def call(self, inputs): # Defines the computation from inputs to outputs 

201 return tf.matmul(inputs, self.w) + self.b 

202 

203 # Instantiates the layer. 

204 linear_layer = SimpleDense(4) 

205 

206 # This will also call `build(input_shape)` and create the weights. 

207 y = linear_layer(tf.ones((2, 2))) 

208 assert len(linear_layer.weights) == 2 

209 

210 # These weights are trainable, so they're listed in `trainable_weights`: 

211 assert len(linear_layer.trainable_weights) == 2 

212 ``` 

213 

214 Note that the method `add_weight()` offers a shortcut to create weights: 

215 

216 ```python 

217 class SimpleDense(Layer): 

218 

219 def __init__(self, units=32): 

220 super(SimpleDense, self).__init__() 

221 self.units = units 

222 

223 def build(self, input_shape): 

224 self.w = self.add_weight(shape=(input_shape[-1], self.units), 

225 initializer='random_normal', 

226 trainable=True) 

227 self.b = self.add_weight(shape=(self.units,), 

228 initializer='random_normal', 

229 trainable=True) 

230 

231 def call(self, inputs): 

232 return tf.matmul(inputs, self.w) + self.b 

233 ``` 

234 

235 Besides trainable weights, updated via backpropagation during training, 

236 layers can also have non-trainable weights. These weights are meant to 

237 be updated manually during `call()`. Here's a example layer that computes 

238 the running sum of its inputs: 

239 

240 ```python 

241 class ComputeSum(Layer): 

242 

243 def __init__(self, input_dim): 

244 super(ComputeSum, self).__init__() 

245 # Create a non-trainable weight. 

246 self.total = tf.Variable(initial_value=tf.zeros((input_dim,)), 

247 trainable=False) 

248 

249 def call(self, inputs): 

250 self.total.assign_add(tf.reduce_sum(inputs, axis=0)) 

251 return self.total 

252 

253 my_sum = ComputeSum(2) 

254 x = tf.ones((2, 2)) 

255 

256 y = my_sum(x) 

257 print(y.numpy()) # [2. 2.] 

258 

259 y = my_sum(x) 

260 print(y.numpy()) # [4. 4.] 

261 

262 assert my_sum.weights == [my_sum.total] 

263 assert my_sum.non_trainable_weights == [my_sum.total] 

264 assert my_sum.trainable_weights == [] 

265 ``` 

266 

267 For more information about creating layers, see the guide 

268 [Making new Layers and Models via subclassing]( 

269 https://www.tensorflow.org/guide/keras/custom_layers_and_models) 

270 """ 

271 

272 # See tf.Module for the usage of this property. 

273 # The key for _obj_reference_counts_dict is a Trackable, which could be a 

274 # variable or layer etc. tf.Module._flatten will fail to flatten the key 

275 # since it is trying to convert Trackable to a string. This attribute can be 

276 # ignored even after the fix of nest lib, since the trackable object should 

277 # already been available as individual attributes. _obj_reference_counts_dict 

278 # just contains a copy of them. 

279 _TF_MODULE_IGNORED_PROPERTIES = frozenset(itertools.chain( 

280 ('_obj_reference_counts_dict',), 

281 module.Module._TF_MODULE_IGNORED_PROPERTIES 

282 )) 

283 

284 # When loading from a SavedModel, Layers typically can be revived into a 

285 # generic Layer wrapper. Sometimes, however, layers may implement methods 

286 # that go beyond this wrapper, as in the case of PreprocessingLayers' 

287 # `adapt` method. When this is the case, layer implementers can override 

288 # must_restore_from_config to return True; layers with this property must 

289 # be restored into their actual objects (and will fail if the object is 

290 # not available to the restoration code). 

291 _must_restore_from_config = False 

292 

293 def _get_cell_name(self): 

294 canonical_name = get_canonical_name_for_symbol( 

295 self.__class__, api_name='keras', add_prefix_to_v1_names=True) 

296 if canonical_name is not None: 

297 return 'tf.{}'.format(canonical_name) 

298 return self.__class__.__module__ + '.' + self.__class__.__name__ 

299 

300 def _instrument_layer_creation(self): 

301 self._instrumented_keras_api = False 

302 self._instrumented_keras_layer_class = False 

303 self._instrumented_keras_model_class = False 

304 if not getattr(self, '_disable_keras_instrumentation', False): 

305 self._instrumented_keras_api = True 

306 if getattr(self, '_is_model_for_instrumentation', False): 

307 self._instrumented_keras_model_class = True 

308 else: 

309 self._instrumented_keras_layer_class = True 

310 

311 @trackable.no_automatic_dependency_tracking 

312 def __init__(self, 

313 trainable=True, 

314 name=None, 

315 dtype=None, 

316 dynamic=False, 

317 **kwargs): 

318 self._instrument_layer_creation() 

319 

320 # These properties should be set by the user via keyword arguments. 

321 # note that 'dtype', 'input_shape' and 'batch_input_shape' 

322 # are only applicable to input layers: do not pass these keywords 

323 # to non-input layers. 

324 allowed_kwargs = { 

325 'input_dim', 

326 'input_shape', 

327 'batch_input_shape', 

328 'batch_size', 

329 'weights', 

330 'activity_regularizer', 

331 'autocast', 

332 'implementation', 

333 } 

334 # Validate optional keyword arguments. 

335 generic_utils.validate_kwargs(kwargs, allowed_kwargs) 

336 

337 # Mutable properties 

338 # Indicates whether the layer's weights are updated during training 

339 # and whether the layer's updates are run during training. 

340 self._trainable = trainable 

341 # A stateful layer is a layer whose updates are run during inference too, 

342 # for instance stateful RNNs. 

343 self._stateful = False 

344 # Indicates whether `build` needs to be called upon layer call, to create 

345 # the layer's weights. 

346 self.built = False 

347 # Provides information about which inputs are compatible with the layer. 

348 self._input_spec = None 

349 

350 # SavedModel-related attributes. 

351 # Record the build input shape for loading purposes. 

352 # TODO(kathywu): Move this to Layer._set_save_spec once cl/290121460 is 

353 # submitted. 

354 self._build_input_shape = None 

355 self._saved_model_inputs_spec = None 

356 

357 # `Layer.compute_mask` will be called at the end of `Layer.__call__` if 

358 # `Layer.compute_mask` is overridden, or if the `Layer` subclass sets 

359 # `self.supports_masking=True`. 

360 self._supports_masking = not generic_utils.is_default(self.compute_mask) 

361 

362 self._init_set_name(name) 

363 self._activity_regularizer = regularizers.get( 

364 kwargs.pop('activity_regularizer', None)) 

365 self._maybe_create_attribute('_trainable_weights', []) 

366 self._maybe_create_attribute('_non_trainable_weights', []) 

367 self._updates = [] 

368 # Object to store all thread local layer properties. 

369 self._thread_local = threading.local() 

370 # A list of zero-argument lambdas which return Tensors, used for variable 

371 # regularizers. 

372 self._callable_losses = [] 

373 # A list of symbolic Tensors containing activity regularizers and losses 

374 # manually added through `add_loss` in graph-building mode. 

375 self._losses = [] 

376 # A list of metric instances corresponding to the symbolic metric tensors 

377 # added using the `add_metric` API. 

378 self._metrics = [] 

379 # Ensures the same metric is not added multiple times in `MirroredStrategy`. 

380 self._metrics_lock = threading.Lock() 

381 

382 # Both graph and subclassed networks have a dtype policy. For graph 

383 # networks, the policy's compute and variable dtypes are ignored. Such 

384 # networks only use the policy if it is a PolicyV1, in which case it uses 

385 # the PolicyV1's loss_scale (Policy does not have a loss_scale). For 

386 # subclassed networks, the compute and variable dtypes are used as like any 

387 # ordinary layer. 

388 self._set_dtype_policy(dtype) 

389 # Boolean indicating whether the layer automatically casts its inputs to the 

390 # layer's compute_dtype. 

391 self._autocast = kwargs.get('autocast', 

392 base_layer_utils.v2_dtype_behavior_enabled()) 

393 

394 # Tracks `TrackableDataStructure`s, `Module`s, and `Layer`s. 

395 # Ordered by when the object was assigned as an attr. 

396 # Entries are unique. 

397 self._maybe_create_attribute('_self_tracked_trackables', []) 

398 

399 # These lists will be filled via successive calls 

400 # to self._add_inbound_node(). 

401 # Used in symbolic mode only, only in conjunction with graph-networks 

402 self._inbound_nodes_value = [] 

403 self._outbound_nodes_value = [] 

404 

405 self._init_call_fn_args() 

406 

407 # Whether the `call` method can be used to build a TF graph without issues. 

408 # This attribute has no effect if the model is created using the Functional 

409 # API. Instead, `model.dynamic` is determined based on the internal layers. 

410 self._dynamic = dynamic 

411 

412 # Manage input shape information if passed. 

413 if 'input_dim' in kwargs and 'input_shape' not in kwargs: 

414 # Backwards compatibility: alias 'input_dim' to 'input_shape'. 

415 kwargs['input_shape'] = (kwargs['input_dim'],) 

416 if 'input_shape' in kwargs or 'batch_input_shape' in kwargs: 

417 # In this case we will later create an input layer 

418 # to insert before the current layer 

419 if 'batch_input_shape' in kwargs: 

420 batch_input_shape = tuple(kwargs['batch_input_shape']) 

421 elif 'input_shape' in kwargs: 

422 if 'batch_size' in kwargs: 

423 batch_size = kwargs['batch_size'] 

424 else: 

425 batch_size = None 

426 batch_input_shape = (batch_size,) + tuple(kwargs['input_shape']) 

427 self._batch_input_shape = batch_input_shape 

428 

429 # Manage initial weight values if passed. 

430 self._initial_weights = kwargs.get('weights', None) 

431 

432 # Whether the layer will track any layers that is set as attribute on itself 

433 # as sub-layers, the weights from the sub-layers will be included in the 

434 # parent layer's variables() as well. 

435 # Default to True, which means auto tracking is turned on. Certain subclass 

436 # might want to turn it off, like Sequential model. 

437 self._auto_track_sub_layers = True 

438 

439 # For backwards compat reasons, most built-in layers do not guarantee 

440 # That they will 100% preserve the structure of input args when saving 

441 # / loading configs. E.g. they may un-nest an arg that is 

442 # a list with one element. 

443 self._preserve_input_structure_in_config = False 

444 

445 @trackable.no_automatic_dependency_tracking 

446 @generic_utils.default 

447 def build(self, input_shape): 

448 """Creates the variables of the layer (optional, for subclass implementers). 

449 

450 This is a method that implementers of subclasses of `Layer` or `Model` 

451 can override if they need a state-creation step in-between 

452 layer instantiation and layer call. 

453 

454 This is typically used to create the weights of `Layer` subclasses. 

455 

456 Args: 

457 input_shape: Instance of `TensorShape`, or list of instances of 

458 `TensorShape` if the layer expects a list of inputs 

459 (one instance per input). 

460 """ 

461 # Only record the build input shapes of overridden build methods. 

462 if not hasattr(self.build, '_is_default'): 

463 self._build_input_shape = input_shape 

464 self.built = True 

465 

466 @doc_controls.for_subclass_implementers 

467 def call(self, inputs, *args, **kwargs): # pylint: disable=unused-argument 

468 """This is where the layer's logic lives. 

469 

470 Note here that `call()` method in `tf.keras` is little bit different 

471 from `keras` API. In `keras` API, you can pass support masking for 

472 layers as additional arguments. Whereas `tf.keras` has `compute_mask()` 

473 method to support masking. 

474 

475 Args: 

476 inputs: Input tensor, or dict/list/tuple of input tensors. 

477 The first positional `inputs` argument is subject to special rules: 

478 - `inputs` must be explicitly passed. A layer cannot have zero 

479 arguments, and `inputs` cannot be provided via the default value 

480 of a keyword argument. 

481 - NumPy array or Python scalar values in `inputs` get cast as tensors. 

482 - Keras mask metadata is only collected from `inputs`. 

483 - Layers are built (`build(input_shape)` method) 

484 using shape info from `inputs` only. 

485 - `input_spec` compatibility is only checked against `inputs`. 

486 - Mixed precision input casting is only applied to `inputs`. 

487 If a layer has tensor arguments in `*args` or `**kwargs`, their 

488 casting behavior in mixed precision should be handled manually. 

489 - The SavedModel input specification is generated using `inputs` only. 

490 - Integration with various ecosystem packages like TFMOT, TFLite, 

491 TF.js, etc is only supported for `inputs` and not for tensors in 

492 positional and keyword arguments. 

493 *args: Additional positional arguments. May contain tensors, although 

494 this is not recommended, for the reasons above. 

495 **kwargs: Additional keyword arguments. May contain tensors, although 

496 this is not recommended, for the reasons above. 

497 The following optional keyword arguments are reserved: 

498 - `training`: Boolean scalar tensor of Python boolean indicating 

499 whether the `call` is meant for training or inference. 

500 - `mask`: Boolean input mask. If the layer's `call()` method takes a 

501 `mask` argument, its default value will be set to the mask generated 

502 for `inputs` by the previous layer (if `input` did come from a layer 

503 that generated a corresponding mask, i.e. if it came from a Keras 

504 layer with masking support). 

505 

506 Returns: 

507 A tensor or list/tuple of tensors. 

508 """ 

509 return inputs 

510 

511 @doc_controls.for_subclass_implementers 

512 def _add_trackable(self, trackable_object, trainable): 

513 """Adds a Trackable object to this layer's state. 

514 

515 Args: 

516 trackable_object: The tf.tracking.Trackable object to add. 

517 trainable: Boolean, whether the variable should be part of the layer's 

518 "trainable_variables" (e.g. variables, biases) or 

519 "non_trainable_variables" (e.g. BatchNorm mean and variance). 

520 

521 Returns: 

522 The TrackableWeightHandler used to track this object. 

523 """ 

524 if isinstance(trackable_object, base_layer_utils.TrackableWeightHandler): 

525 handler = trackable_object 

526 else: 

527 handler = base_layer_utils.TrackableWeightHandler(trackable_object) 

528 if trainable: 

529 self._trainable_weights.append(handler) 

530 else: 

531 self._non_trainable_weights.append(handler) 

532 return handler 

533 

534 @doc_controls.for_subclass_implementers 

535 def add_weight(self, 

536 name=None, 

537 shape=None, 

538 dtype=None, 

539 initializer=None, 

540 regularizer=None, 

541 trainable=None, 

542 constraint=None, 

543 use_resource=None, 

544 synchronization=tf_variables.VariableSynchronization.AUTO, 

545 aggregation=tf_variables.VariableAggregation.NONE, 

546 **kwargs): 

547 """Adds a new variable to the layer. 

548 

549 Args: 

550 name: Variable name. 

551 shape: Variable shape. Defaults to scalar if unspecified. 

552 dtype: The type of the variable. Defaults to `self.dtype`. 

553 initializer: Initializer instance (callable). 

554 regularizer: Regularizer instance (callable). 

555 trainable: Boolean, whether the variable should be part of the layer's 

556 "trainable_variables" (e.g. variables, biases) 

557 or "non_trainable_variables" (e.g. BatchNorm mean and variance). 

558 Note that `trainable` cannot be `True` if `synchronization` 

559 is set to `ON_READ`. 

560 constraint: Constraint instance (callable). 

561 use_resource: Whether to use `ResourceVariable`. 

562 synchronization: Indicates when a distributed a variable will be 

563 aggregated. Accepted values are constants defined in the class 

564 `tf.VariableSynchronization`. By default the synchronization is set to 

565 `AUTO` and the current `DistributionStrategy` chooses 

566 when to synchronize. If `synchronization` is set to `ON_READ`, 

567 `trainable` must not be set to `True`. 

568 aggregation: Indicates how a distributed variable will be aggregated. 

569 Accepted values are constants defined in the class 

570 `tf.VariableAggregation`. 

571 **kwargs: Additional keyword arguments. Accepted values are `getter`, 

572 `collections`, `experimental_autocast` and `caching_device`. 

573 

574 Returns: 

575 The variable created. 

576 

577 Raises: 

578 ValueError: When giving unsupported dtype and no initializer or when 

579 trainable has been set to True with synchronization set as `ON_READ`. 

580 """ 

581 if shape is None: 

582 shape = () 

583 kwargs.pop('partitioner', None) # Ignored. 

584 # Validate optional keyword arguments. 

585 for kwarg in kwargs: 

586 if kwarg not in ['collections', 'experimental_autocast', 

587 'caching_device', 'getter']: 

588 raise TypeError('Unknown keyword argument:', kwarg) 

589 collections_arg = kwargs.pop('collections', None) 

590 # 'experimental_autocast' can be set to False by the caller to indicate an 

591 # AutoCastVariable should never be created. 

592 autocast = kwargs.pop('experimental_autocast', True) 

593 # See the docstring for tf.Variable about the details for caching_device. 

594 caching_device = kwargs.pop('caching_device', None) 

595 

596 if dtype is None: 

597 dtype = self.dtype or backend.floatx() 

598 dtype = dtypes.as_dtype(dtype) 

599 if self._dtype_policy.variable_dtype is None: 

600 # The policy is "_infer", so we infer the policy from the variable dtype. 

601 self._set_dtype_policy(policy.Policy(dtype.base_dtype.name)) 

602 initializer = initializers.get(initializer) 

603 regularizer = regularizers.get(regularizer) 

604 constraint = constraints.get(constraint) 

605 

606 if synchronization == tf_variables.VariableSynchronization.ON_READ: 

607 if trainable: 

608 raise ValueError( 

609 'Synchronization value can be set to ' 

610 'VariableSynchronization.ON_READ only for non-trainable variables. ' 

611 'You have specified trainable=True and ' 

612 'synchronization=VariableSynchronization.ON_READ.') 

613 else: 

614 # Set trainable to be false when variable is to be synced on read. 

615 trainable = False 

616 elif trainable is None: 

617 trainable = True 

618 

619 # Initialize variable when no initializer provided 

620 if initializer is None: 

621 # If dtype is DT_FLOAT, provide a uniform unit scaling initializer 

622 if dtype.is_floating: 

623 initializer = initializers.get('glorot_uniform') 

624 # If dtype is DT_INT/DT_UINT, provide a default value `zero` 

625 # If dtype is DT_BOOL, provide a default value `FALSE` 

626 elif dtype.is_integer or dtype.is_unsigned or dtype.is_bool: 

627 initializer = initializers.get('zeros') 

628 # NOTES:Do we need to support for handling DT_STRING and DT_COMPLEX here? 

629 elif 'getter' not in kwargs: 

630 # When `getter` is specified, it's possibly fine for `initializer` to be 

631 # None since it's up to the custom `getter` to raise error in case it 

632 # indeed needs `initializer`. 

633 raise ValueError('An initializer for variable %s of type %s is required' 

634 ' for layer %s' % (name, dtype.base_dtype, self.name)) 

635 

636 getter = kwargs.pop('getter', base_layer_utils.make_variable) 

637 if (autocast and 

638 self._dtype_policy.compute_dtype != self._dtype_policy.variable_dtype 

639 and dtype.is_floating): 

640 old_getter = getter 

641 # Wrap variable constructor to return an AutoCastVariable. 

642 def getter(*args, **kwargs): # pylint: disable=function-redefined 

643 variable = old_getter(*args, **kwargs) 

644 return autocast_variable.create_autocast_variable(variable) 

645 # Also the caching_device does not work with the mixed precision API, 

646 # disable it if it is specified. 

647 # TODO(b/142020079): Reenable it once the bug is fixed. 

648 if caching_device is not None: 

649 tf_logging.warning( 

650 '`caching_device` does not work with mixed precision API. Ignoring ' 

651 'user specified `caching_device`.') 

652 caching_device = None 

653 

654 variable = self._add_variable_with_custom_getter( 

655 name=name, 

656 shape=shape, 

657 # TODO(allenl): a `make_variable` equivalent should be added as a 

658 # `Trackable` method. 

659 getter=getter, 

660 # Manage errors in Layer rather than Trackable. 

661 overwrite=True, 

662 initializer=initializer, 

663 dtype=dtype, 

664 constraint=constraint, 

665 trainable=trainable, 

666 use_resource=use_resource, 

667 collections=collections_arg, 

668 synchronization=synchronization, 

669 aggregation=aggregation, 

670 caching_device=caching_device) 

671 if regularizer is not None: 

672 # TODO(fchollet): in the future, this should be handled at the 

673 # level of variable creation, and weight regularization losses 

674 # should be variable attributes. 

675 name_in_scope = variable.name[:variable.name.find(':')] 

676 self._handle_weight_regularization(name_in_scope, 

677 variable, 

678 regularizer) 

679 if base_layer_utils.is_split_variable(variable): 

680 for v in variable: 

681 backend.track_variable(v) 

682 if trainable: 

683 self._trainable_weights.append(v) 

684 else: 

685 self._non_trainable_weights.append(v) 

686 else: 

687 backend.track_variable(variable) 

688 if trainable: 

689 self._trainable_weights.append(variable) 

690 else: 

691 self._non_trainable_weights.append(variable) 

692 return variable 

693 

694 @generic_utils.default 

695 def get_config(self): 

696 """Returns the config of the layer. 

697 

698 A layer config is a Python dictionary (serializable) 

699 containing the configuration of a layer. 

700 The same layer can be reinstantiated later 

701 (without its trained weights) from this configuration. 

702 

703 The config of a layer does not include connectivity 

704 information, nor the layer class name. These are handled 

705 by `Network` (one layer of abstraction above). 

706 

707 Note that `get_config()` does not guarantee to return a fresh copy of dict 

708 every time it is called. The callers should make a copy of the returned dict 

709 if they want to modify it. 

710 

711 Returns: 

712 Python dictionary. 

713 """ 

714 all_args = tf_inspect.getfullargspec(self.__init__).args 

715 config = { 

716 'name': self.name, 

717 'trainable': self.trainable, 

718 } 

719 if hasattr(self, '_batch_input_shape'): 

720 config['batch_input_shape'] = self._batch_input_shape 

721 config['dtype'] = policy.serialize(self._dtype_policy) 

722 if hasattr(self, 'dynamic'): 

723 # Only include `dynamic` in the `config` if it is `True` 

724 if self.dynamic: 

725 config['dynamic'] = self.dynamic 

726 elif 'dynamic' in all_args: 

727 all_args.remove('dynamic') 

728 expected_args = config.keys() 

729 # Finds all arguments in the `__init__` that are not in the config: 

730 extra_args = [arg for arg in all_args if arg not in expected_args] 

731 # Check that either the only argument in the `__init__` is `self`, 

732 # or that `get_config` has been overridden: 

733 if len(extra_args) > 1 and hasattr(self.get_config, '_is_default'): 

734 raise NotImplementedError('Layer %s has arguments in `__init__` and ' 

735 'therefore must override `get_config`.' % 

736 self.__class__.__name__) 

737 return config 

738 

739 @classmethod 

740 def from_config(cls, config): 

741 """Creates a layer from its config. 

742 

743 This method is the reverse of `get_config`, 

744 capable of instantiating the same layer from the config 

745 dictionary. It does not handle layer connectivity 

746 (handled by Network), nor weights (handled by `set_weights`). 

747 

748 Args: 

749 config: A Python dictionary, typically the 

750 output of get_config. 

751 

752 Returns: 

753 A layer instance. 

754 """ 

755 return cls(**config) 

756 

757 def compute_output_shape(self, input_shape): 

758 """Computes the output shape of the layer. 

759 

760 If the layer has not been built, this method will call `build` on the 

761 layer. This assumes that the layer will later be used with inputs that 

762 match the input shape provided here. 

763 

764 Args: 

765 input_shape: Shape tuple (tuple of integers) 

766 or list of shape tuples (one per output tensor of the layer). 

767 Shape tuples can include None for free dimensions, 

768 instead of an integer. 

769 

770 Returns: 

771 An input shape tuple. 

772 """ 

773 if context.executing_eagerly(): 

774 # In this case we build the model first in order to do shape inference. 

775 # This is acceptable because the framework only calls 

776 # `compute_output_shape` on shape values that the layer would later be 

777 # built for. It would however cause issues in case a user attempts to 

778 # use `compute_output_shape` manually with shapes that are incompatible 

779 # with the shape the Layer will be called on (these users will have to 

780 # implement `compute_output_shape` themselves). 

781 self._maybe_build(input_shape) 

782 with func_graph.FuncGraph(str(self.name) + '_scratch_graph').as_default(): 

783 input_shape = tf_utils.convert_shapes(input_shape, to_tuples=False) 

784 def _make_placeholder_like(shape): 

785 ph = backend.placeholder(shape=shape, dtype=self.dtype) 

786 ph._keras_mask = None 

787 return ph 

788 inputs = nest.map_structure(_make_placeholder_like, input_shape) 

789 try: 

790 outputs = self(inputs, training=False) 

791 except TypeError as e: 

792 raise NotImplementedError( 

793 'We could not automatically infer the static shape of the ' 

794 'layer\'s output. Please implement the ' 

795 '`compute_output_shape` method on your layer (%s).' % 

796 self.__class__.__name__) from e 

797 return nest.map_structure(lambda t: t.shape, outputs) 

798 raise NotImplementedError( 

799 'Please run in eager mode or implement the `compute_output_shape` ' 

800 'method on your layer (%s).' % self.__class__.__name__) 

801 

802 @doc_controls.for_subclass_implementers 

803 def compute_output_signature(self, input_signature): 

804 """Compute the output tensor signature of the layer based on the inputs. 

805 

806 Unlike a TensorShape object, a TensorSpec object contains both shape 

807 and dtype information for a tensor. This method allows layers to provide 

808 output dtype information if it is different from the input dtype. 

809 For any layer that doesn't implement this function, 

810 the framework will fall back to use `compute_output_shape`, and will 

811 assume that the output dtype matches the input dtype. 

812 

813 Args: 

814 input_signature: Single TensorSpec or nested structure of TensorSpec 

815 objects, describing a candidate input for the layer. 

816 

817 Returns: 

818 Single TensorSpec or nested structure of TensorSpec objects, describing 

819 how the layer would transform the provided input. 

820 

821 Raises: 

822 TypeError: If input_signature contains a non-TensorSpec object. 

823 """ 

824 def check_type_return_shape(s): 

825 if not isinstance(s, tensor_spec.TensorSpec): 

826 raise TypeError('Only TensorSpec signature types are supported, ' 

827 'but saw signature entry: {}.'.format(s)) 

828 return s.shape 

829 input_shape = nest.map_structure(check_type_return_shape, input_signature) 

830 output_shape = self.compute_output_shape(input_shape) 

831 dtype = self._compute_dtype 

832 if dtype is None: 

833 input_dtypes = [s.dtype for s in nest.flatten(input_signature)] 

834 # Default behavior when self.dtype is None, is to use the first input's 

835 # dtype. 

836 dtype = input_dtypes[0] 

837 return nest.map_structure( 

838 lambda s: tensor_spec.TensorSpec(dtype=dtype, shape=s), 

839 output_shape) 

840 

841 def _keras_tensor_symbolic_call(self, inputs, input_masks, args, kwargs): 

842 if self.dynamic: 

843 # We will use static shape inference to return symbolic tensors 

844 # matching the specifications of the layer outputs. 

845 # Since `self.dynamic` is True, we will never attempt to 

846 # run the underlying TF graph (which is disconnected). 

847 # TODO(fchollet): consider py_func as an alternative, which 

848 # would enable us to run the underlying graph if needed. 

849 input_signature = nest.map_structure( 

850 lambda x: tensor_spec.TensorSpec(shape=x.shape, dtype=x.dtype), 

851 inputs) 

852 output_signature = self.compute_output_signature(input_signature) 

853 return nest.map_structure(keras_tensor.KerasTensor, output_signature) 

854 else: 

855 return self._infer_output_signature(inputs, args, kwargs, input_masks) 

856 

857 def _infer_output_signature(self, inputs, args, kwargs, input_masks): 

858 """TODO(kaftan): Docstring.""" 

859 

860 call_fn = self.call 

861 # Wrapping `call` function in autograph to allow for dynamic control 

862 # flow and control dependencies in call. We are limiting this to 

863 # subclassed layers as autograph is strictly needed only for 

864 # subclassed layers and models. 

865 # tf_convert will respect the value of autograph setting in the 

866 # enclosing tf.function, if any. 

867 if (base_layer_utils.is_subclassed(self) and 

868 not base_layer_utils.from_saved_model(self)): 

869 call_fn = autograph.tf_convert(self.call, ag_ctx.control_status_ctx()) 

870 

871 # We enter a scratch graph and build placeholder inputs inside of it that 

872 # match the input args. 

873 # We then call the layer inside of the scratch graph to identify the 

874 # output signatures, then we build KerasTensors corresponding to those 

875 # outputs. 

876 scratch_graph = func_graph.FuncGraph(str(self.name) + '_scratch_graph') 

877 with scratch_graph.as_default(): 

878 inputs = nest.map_structure( 

879 keras_tensor.keras_tensor_to_placeholder, inputs) 

880 args = nest.map_structure( 

881 keras_tensor.keras_tensor_to_placeholder, args) 

882 kwargs = nest.map_structure( 

883 keras_tensor.keras_tensor_to_placeholder, kwargs) 

884 input_masks = nest.map_structure( 

885 keras_tensor.keras_tensor_to_placeholder, input_masks) 

886 

887 with backend.name_scope(self._name_scope()): # pylint: disable=not-callable 

888 with autocast_variable.enable_auto_cast_variables( 

889 self._compute_dtype_object): 

890 # Build layer if applicable (if the `build` method has been 

891 # overridden). 

892 # TODO(kaftan): do we maybe_build here, or have we already done it? 

893 self._maybe_build(inputs) 

894 inputs = self._maybe_cast_inputs(inputs) 

895 outputs = call_fn(inputs, *args, **kwargs) 

896 

897 self._handle_activity_regularization(inputs, outputs) 

898 self._set_mask_metadata(inputs, outputs, input_masks, 

899 build_graph=False) 

900 outputs = nest.map_structure( 

901 keras_tensor.keras_tensor_from_tensor, outputs) 

902 

903 if hasattr(self, '_set_inputs') and not self.inputs: 

904 # TODO(kaftan): figure out if we need to do this at all 

905 # Subclassed network: explicitly set metadata normally set by 

906 # a call to self._set_inputs(). 

907 self._set_inputs(inputs, outputs) 

908 del scratch_graph 

909 return outputs 

910 

911 @generic_utils.default 

912 def compute_mask(self, inputs, mask=None): # pylint: disable=unused-argument 

913 """Computes an output mask tensor. 

914 

915 Args: 

916 inputs: Tensor or list of tensors. 

917 mask: Tensor or list of tensors. 

918 

919 Returns: 

920 None or a tensor (or list of tensors, 

921 one per output tensor of the layer). 

922 """ 

923 if not self._supports_masking: 

924 if any(m is not None for m in nest.flatten(mask)): 

925 raise TypeError('Layer ' + self.name + ' does not support masking, ' 

926 'but was passed an input_mask: ' + str(mask)) 

927 # masking not explicitly supported: return None as mask. 

928 return None 

929 # if masking is explicitly supported, by default 

930 # carry over the input mask 

931 return mask 

932 

933 def __call__(self, *args, **kwargs): 

934 """Wraps `call`, applying pre- and post-processing steps. 

935 

936 Args: 

937 *args: Positional arguments to be passed to `self.call`. 

938 **kwargs: Keyword arguments to be passed to `self.call`. 

939 

940 Returns: 

941 Output tensor(s). 

942 

943 Note: 

944 - The following optional keyword arguments are reserved for specific uses: 

945 * `training`: Boolean scalar tensor of Python boolean indicating 

946 whether the `call` is meant for training or inference. 

947 * `mask`: Boolean input mask. 

948 - If the layer's `call` method takes a `mask` argument (as some Keras 

949 layers do), its default value will be set to the mask generated 

950 for `inputs` by the previous layer (if `input` did come from 

951 a layer that generated a corresponding mask, i.e. if it came from 

952 a Keras layer with masking support. 

953 - If the layer is not built, the method will call `build`. 

954 

955 Raises: 

956 ValueError: if the layer's `call` method returns None (an invalid value). 

957 RuntimeError: if `super().__init__()` was not called in the constructor. 

958 """ 

959 if not hasattr(self, '_thread_local'): 

960 raise RuntimeError( 

961 'You must call `super().__init__()` in the layer constructor.') 

962 

963 # `inputs` (the first arg in the method spec) is special cased in 

964 # layer call due to historical reasons. 

965 # This special casing currently takes the form of: 

966 # - 'inputs' must be explicitly passed. A layer cannot have zero arguments, 

967 # and inputs cannot have been provided via the default value of a kwarg. 

968 # - numpy/scalar values in `inputs` get converted to tensors 

969 # - implicit masks / mask metadata are only collected from 'inputs` 

970 # - Layers are built using shape info from 'inputs' only 

971 # - input_spec compatibility is only checked against `inputs` 

972 # - mixed precision casting (autocast) is only applied to `inputs`, 

973 # not to any other argument. 

974 # - setting the SavedModel saving spec. 

975 inputs, args, kwargs = self._split_out_first_arg(args, kwargs) 

976 input_list = nest.flatten(inputs) 

977 

978 # Functional Model construction mode is invoked when `Layer`s are called on 

979 # symbolic `KerasTensor`s, i.e.: 

980 # >> inputs = tf.keras.Input(10) 

981 # >> outputs = MyLayer()(inputs) # Functional construction mode. 

982 # >> model = tf.keras.Model(inputs, outputs) 

983 if _in_functional_construction_mode(self, inputs, args, kwargs, input_list): 

984 return self._functional_construction_call(inputs, args, kwargs, 

985 input_list) 

986 

987 # Maintains info about the `Layer.call` stack. 

988 call_context = base_layer_utils.call_context()