Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/tensorflow/python/keras/engine/sequential.py: 20%

3# Licensed under the Apache License, Version 2.0 (the "License");

4# you may not use this file except in compliance with the License.

5# You may obtain a copy of the License at

7# http://www.apache.org/licenses/LICENSE-2.0

9# Unless required by applicable law or agreed to in writing, software

10# distributed under the License is distributed on an "AS IS" BASIS,

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

12# See the License for the specific language governing permissions and

13# limitations under the License.

14# ==============================================================================

15# pylint: disable=protected-access

16"""Home of the `Sequential` model."""

18import copy

19import warnings

21from tensorflow.python import tf2

22from tensorflow.python.framework import ops

23from tensorflow.python.framework import tensor_util

24from tensorflow.python.keras import layers as layer_module

25from tensorflow.python.keras.engine import base_layer

26from tensorflow.python.keras.engine import functional

27from tensorflow.python.keras.engine import input_layer

28from tensorflow.python.keras.engine import training_utils

29from tensorflow.python.keras.saving.saved_model import model_serialization

30from tensorflow.python.keras.utils import generic_utils

31from tensorflow.python.keras.utils import layer_utils

32from tensorflow.python.keras.utils import tf_inspect

33from tensorflow.python.keras.utils import tf_utils

34from tensorflow.python.module import module

35from tensorflow.python.ops.numpy_ops import np_arrays

36from tensorflow.python.platform import tf_logging as logging

37from tensorflow.python.trackable import base as trackable

38from tensorflow.python.util import nest

39from tensorflow.python.util.tf_export import keras_export

42SINGLE_LAYER_OUTPUT_ERROR_MSG = ('All layers in a Sequential model should have '

43 'a single output tensor. For multi-output '

44 'layers, use the functional API.')

47@keras_export('keras.Sequential', 'keras.models.Sequential')

48class Sequential(functional.Functional):

49 """`Sequential` groups a linear stack of layers into a `tf.keras.Model`.

51 `Sequential` provides training and inference features on this model.

53 Examples:

55 >>> # Optionally, the first layer can receive an `input_shape` argument:

56 >>> model = tf.keras.Sequential()

57 >>> model.add(tf.keras.layers.Dense(8, input_shape=(16,)))

58 >>> # Afterwards, we do automatic shape inference:

59 >>> model.add(tf.keras.layers.Dense(4))

61 >>> # This is identical to the following:

62 >>> model = tf.keras.Sequential()

63 >>> model.add(tf.keras.Input(shape=(16,)))

64 >>> model.add(tf.keras.layers.Dense(8))

66 >>> # Note that you can also omit the `input_shape` argument.

67 >>> # In that case the model doesn't have any weights until the first call

68 >>> # to a training/evaluation method (since it isn't yet built):

69 >>> model = tf.keras.Sequential()

70 >>> model.add(tf.keras.layers.Dense(8))

71 >>> model.add(tf.keras.layers.Dense(4))

72 >>> # model.weights not created yet

74 >>> # Whereas if you specify the input shape, the model gets built

75 >>> # continuously as you are adding layers:

76 >>> model = tf.keras.Sequential()

77 >>> model.add(tf.keras.layers.Dense(8, input_shape=(16,)))

78 >>> model.add(tf.keras.layers.Dense(4))

79 >>> len(model.weights)

80 4

82 >>> # When using the delayed-build pattern (no input shape specified), you can

83 >>> # choose to manually build your model by calling

84 >>> # `build(batch_input_shape)`:

85 >>> model = tf.keras.Sequential()

86 >>> model.add(tf.keras.layers.Dense(8))

87 >>> model.add(tf.keras.layers.Dense(4))

88 >>> model.build((None, 16))

89 >>> len(model.weights)

90 4

92 ```python

93 # Note that when using the delayed-build pattern (no input shape specified),

94 # the model gets built the first time you call `fit`, `eval`, or `predict`,

95 # or the first time you call the model on some input data.

96 model = tf.keras.Sequential()

97 model.add(tf.keras.layers.Dense(8))

98 model.add(tf.keras.layers.Dense(1))

99 model.compile(optimizer='sgd', loss='mse')

100 # This builds the model for the first time:

101 model.fit(x, y, batch_size=32, epochs=10)

102 ```

103 """

104

105 @trackable.no_automatic_dependency_tracking

106 def __init__(self, layers=None, name=None):

107 """Creates a `Sequential` model instance.

108

109 Args:

110 layers: Optional list of layers to add to the model.

111 name: Optional name for the model.

112 """

113 # Skip the init in FunctionalModel since model doesn't have input/output yet

114 super(functional.Functional, self).__init__( # pylint: disable=bad-super-call

115 name=name, autocast=False)

116 self.supports_masking = True

117 self._compute_output_and_mask_jointly = True

118 self._auto_track_sub_layers = False

119 self._inferred_input_shape = None

120 self._has_explicit_input_shape = False

121 self._input_dtype = None

122 self._layer_call_argspecs = {}

123 self._created_nodes = set()

124 # Flag that indicate whether the sequential network topology has been

125 # created. It is false when there isn't any layer, or the layers doesn't

126 # have input shape.

127 self._graph_initialized = False

128

129 # Unfortunately some Sequential models using custom layers or FeatureColumn

130 # layers have multiple inputs. This is fundamentally incompatible with

131 # most of the Sequential API, and we have to disable a number of features

132 # for such models.

133 self._use_legacy_deferred_behavior = False

134

135 # Add to the model any layers passed to the constructor.

136 if layers:

137 if not isinstance(layers, (list, tuple)):

138 layers = [layers]

139 for layer in layers:

140 self.add(layer)

141

142 @property

143 def layers(self):

144 # Historically, `sequential.layers` only returns layers that were added

145 # via `add`, and omits the auto-generated `InputLayer` that comes at the

146 # bottom of the stack.

147 # `Trackable` manages the `_layers` attributes and does filtering

148 # over it.

149 layers = super(Sequential, self).layers

150 if layers and isinstance(layers[0], input_layer.InputLayer):

151 return layers[1:]

152 return layers[:]

153

154 @trackable.no_automatic_dependency_tracking

155 def add(self, layer):

156 """Adds a layer instance on top of the layer stack.

157

158 Args:

159 layer: layer instance.

160

161 Raises:

162 TypeError: If `layer` is not a layer instance.

163 ValueError: In case the `layer` argument does not

164 know its input shape.

165 ValueError: In case the `layer` argument has

166 multiple output tensors, or is already connected

167 somewhere else (forbidden in `Sequential` models).

168 """

169 # If we are passed a Keras tensor created by keras.Input(), we can extract

170 # the input layer from its keras history and use that without any loss of

171 # generality.

172 if hasattr(layer, '_keras_history'):

173 origin_layer = layer._keras_history[0]

174 if isinstance(origin_layer, input_layer.InputLayer):

175 layer = origin_layer

176 logging.warning(

177 'Please add `keras.layers.InputLayer` instead of `keras.Input` to '

178 'Sequential model. `keras.Input` is intended to be used by '

179 'Functional model.')

180

181 if isinstance(layer, module.Module):

182 if not isinstance(layer, base_layer.Layer):

183 layer = functional.ModuleWrapper(layer)

184 else:

185 raise TypeError('The added layer must be '

186 'an instance of class Layer. '

187 'Found: ' + str(layer))

188

189 tf_utils.assert_no_legacy_layers([layer])

190 if not self._is_layer_name_unique(layer):

191 raise ValueError('All layers added to a Sequential model '

192 'should have unique names. Name "%s" is already the name'

193 ' of a layer in this model. Update the `name` argument '

194 'to pass a unique name.' % (layer.name,))

195

196 self.built = False

197 set_inputs = False

198 self._maybe_create_attribute('_self_tracked_trackables', [])

199 if not self._self_tracked_trackables:

200 if isinstance(layer, input_layer.InputLayer):

201 # Case where the user passes an Input or InputLayer layer via `add`.

202 set_inputs = True

203 else:

204 batch_shape, dtype = training_utils.get_input_shape_and_dtype(layer)

205 if batch_shape:

206 # Instantiate an input layer.

207 x = input_layer.Input(

208 batch_shape=batch_shape, dtype=dtype, name=layer.name + '_input')

209 # This will build the current layer

210 # and create the node connecting the current layer

211 # to the input layer we just created.

212 layer(x)

213 set_inputs = True

214

215 if set_inputs:

216 outputs = nest.flatten(layer._inbound_nodes[-1].outputs)

217 if len(outputs) != 1:

218 raise ValueError(SINGLE_LAYER_OUTPUT_ERROR_MSG)

219 self.outputs = outputs

220 self.inputs = layer_utils.get_source_inputs(self.outputs[0])

221 self.built = True

222 self._has_explicit_input_shape = True

223

224 elif self.outputs:

225 # If the model is being built continuously on top of an input layer:

226 # refresh its output.

227 output_tensor = layer(self.outputs[0])

228 if len(nest.flatten(output_tensor)) != 1:

229 raise ValueError(SINGLE_LAYER_OUTPUT_ERROR_MSG)

230 self.outputs = [output_tensor]

231 self.built = True

232

233 if set_inputs or self._graph_initialized:

234 self._init_graph_network(self.inputs, self.outputs)

235 self._graph_initialized = True

236 else:

237 self._self_tracked_trackables.append(layer)

238 self._handle_deferred_layer_dependencies([layer])

239

240 self._layer_call_argspecs[layer] = tf_inspect.getfullargspec(layer.call)

241

242 @trackable.no_automatic_dependency_tracking

243 def pop(self):

244 """Removes the last layer in the model.

245

246 Raises:

247 TypeError: if there are no layers in the model.

248 """

249 if not self.layers:

250 raise TypeError('There are no layers in the model.')

251

252 layer = self._self_tracked_trackables.pop()

253 self._layer_call_argspecs.pop(layer)

254 if not self.layers:

255 self.outputs = None

256 self.inputs = None

257 self.built = False

258 self._inferred_input_shape = None

259 self._has_explicit_input_shape = False

260 self._graph_initialized = False

261 elif self._graph_initialized:

262 self.layers[-1]._outbound_nodes = []

263 self.outputs = [self.layers[-1].output]

264 self._init_graph_network(self.inputs, self.outputs)

265 self.built = True

266

267 @trackable.no_automatic_dependency_tracking

268 def _build_graph_network_for_inferred_shape(self,

269 input_shape,

270 input_dtype=None):

271 if input_shape is None or not self.layers:

272 return

273 if not tf2.enabled() or not ops.executing_eagerly_outside_functions():

274 # This behavior is disabled in V1 or when eager execution is disabled.

275 return

276 if (not self._has_explicit_input_shape and

277 not self._use_legacy_deferred_behavior):

278 # Determine whether the input shape is novel, i.e. whether the model

279 # should be rebuilt.

280 input_shape = tuple(input_shape)

281 if self._inferred_input_shape is None:

282 new_shape = input_shape

283 else:

284 new_shape = relax_input_shape(self._inferred_input_shape, input_shape)

285 if (new_shape is not None and new_shape != self._inferred_input_shape):

286 # A novel shape has been received: we need to rebuild the model.

287 # In case we are inside a graph function, we step out of it.

288 with ops.init_scope():

289 inputs = input_layer.Input(

290 batch_shape=new_shape,

291 dtype=input_dtype,

292 name=self.layers[0].name + '_input')

293 layer_input = inputs

294 created_nodes = set()

295 for layer in self.layers:

296 # Clear nodes previously created via this method. This prevents

297 # node accumulation and ensures that e.g. `layer.output` is

298 # always connected to `model.inputs`

299 # (this is important e.g. for the feature extraction use case).

300 # We don't just do `layer._inbound_nodes = []` in order

301 # not to break shared layers added to Sequential models (which is

302 # technically illegal as per the `add()` docstring,

303 # but wasn't previously disabled).

304 clear_previously_created_nodes(layer, self._created_nodes)

305 try:

306 # Create Functional API connection by calling the current layer

307 layer_output = layer(layer_input)

308 except: # pylint:disable=bare-except

309 # Functional API calls may fail for a number of reasons:

310 # 1) The layer may be buggy. In this case it will be easier for

311 # the user to debug if we fail on the first call on concrete data,

312 # instead of our own call on a symbolic input.

313 # 2) The layer is dynamic (graph-incompatible) and hasn't

314 # overridden `compute_output_shape`. In this case, it is

315 # impossible to build a graph network.

316 # 3) The layer is otherwise incompatible with the Functional API

317 # (e.g. this is the case for some probabilistic layers that rely

318 # on hacks and that do not return tensors).

319 # In all these cases, we should avoid creating a graph network

320 # (or we simply can't).

321 self._use_legacy_deferred_behavior = True

322 return

323 if len(nest.flatten(layer_output)) != 1:

324 raise ValueError(SINGLE_LAYER_OUTPUT_ERROR_MSG)

325 # Keep track of nodes just created above

326 track_nodes_created_by_last_call(layer, created_nodes)

327 layer_input = layer_output

328 outputs = layer_output

329 self._created_nodes = created_nodes

330 try:

331 # Initialize a graph Network. This call will never fail for

332 # a stack of valid Keras layers.

333 # However some users have layers that are fundamentally incompatible

334 # with the Functional API, which do not return tensors. In this

335 # case, we fall back to the legacy deferred behavior.

336 # TODO(fchollet): consider raising here, as we should not be

337 # supporting such layers.

338 self._init_graph_network(inputs, outputs)

339 self._graph_initialized = True

340 except: # pylint:disable=bare-except

341 self._use_legacy_deferred_behavior = True

342 self._inferred_input_shape = new_shape

343

344 @generic_utils.default

345 def build(self, input_shape=None):

346 if self._graph_initialized:

347 self._init_graph_network(self.inputs, self.outputs)

348 else:

349 if input_shape is None:

350 raise ValueError('You must provide an `input_shape` argument.')

351 self._build_graph_network_for_inferred_shape(input_shape)

352 if not self.built:

353 input_shape = tuple(input_shape)

354 self._build_input_shape = input_shape

355 super(Sequential, self).build(input_shape)

356 self.built = True

357

358 def call(self, inputs, training=None, mask=None): # pylint: disable=redefined-outer-name

359 # If applicable, update the static input shape of the model.

360 if not self._has_explicit_input_shape:

361 if not tensor_util.is_tf_type(inputs) and not isinstance(

362 inputs, np_arrays.ndarray):

363 # This is a Sequential with mutiple inputs. This is technically an

364 # invalid use case of Sequential, but we tolerate it for backwards

365 # compatibility.

366 self._use_legacy_deferred_behavior = True

367 self._build_input_shape = nest.map_structure(_get_shape_tuple, inputs)

368 if tf2.enabled():

369 logging.warning('Layers in a Sequential model should only have a '

370 'single input tensor, but we receive a %s input: %s'

371 '\nConsider rewriting this model with the Functional '

372 'API.' % (type(inputs), inputs))

373 else:

374 self._build_graph_network_for_inferred_shape(inputs.shape, inputs.dtype)

375

376 if self._graph_initialized:

377 if not self.built:

378 self._init_graph_network(self.inputs, self.outputs)

379 return super(Sequential, self).call(inputs, training=training, mask=mask)

380

381 outputs = inputs # handle the corner case where self.layers is empty

382 for layer in self.layers:

383 # During each iteration, `inputs` are the inputs to `layer`, and `outputs`

384 # are the outputs of `layer` applied to `inputs`. At the end of each

385 # iteration `inputs` is set to `outputs` to prepare for the next layer.

386 kwargs = {}

387 argspec = self._layer_call_argspecs[layer].args

388 if 'mask' in argspec:

389 kwargs['mask'] = mask

390 if 'training' in argspec:

391 kwargs['training'] = training

392

393 outputs = layer(inputs, **kwargs)

394

395 if len(nest.flatten(outputs)) != 1:

396 raise ValueError(SINGLE_LAYER_OUTPUT_ERROR_MSG)

397 # `outputs` will be the inputs to the next layer.

398 inputs = outputs

399 mask = getattr(outputs, '_keras_mask', None)

400 return outputs

401

402 def compute_output_shape(self, input_shape):

403 shape = input_shape

404 for layer in self.layers:

405 shape = layer.compute_output_shape(shape)

406 return shape

407

408 def compute_mask(self, inputs, mask):

409 # TODO(omalleyt): b/123540974 This function is not really safe to call

410 # by itself because it will duplicate any updates and losses in graph

411 # mode by `call`ing the Layers again.

412 outputs = self.call(inputs, mask=mask) # pylint: disable=unexpected-keyword-arg

413 return getattr(outputs, '_keras_mask', None)

414

415 def predict_proba(self, x, batch_size=32, verbose=0):

416 """Generates class probability predictions for the input samples.

417

418 The input samples are processed batch by batch.

419

420 Args:

421 x: input data, as a Numpy array or list of Numpy arrays

422 (if the model has multiple inputs).

423 batch_size: integer.

424 verbose: verbosity mode, 0 or 1.

425

426 Returns:

427 A Numpy array of probability predictions.

428 """

429 warnings.warn('`model.predict_proba()` is deprecated and '

430 'will be removed after 2021-01-01. '

431 'Please use `model.predict()` instead.')

432 preds = self.predict(x, batch_size, verbose)

433 if preds.min() < 0. or preds.max() > 1.:

434 logging.warning('Network returning invalid probability values. '

435 'The last layer might not normalize predictions '

436 'into probabilities '

437 '(like softmax or sigmoid would).')

438 return preds

439

440 def predict_classes(self, x, batch_size=32, verbose=0):

441 """Generate class predictions for the input samples.

442

443 The input samples are processed batch by batch.

444

445 Args:

446 x: input data, as a Numpy array or list of Numpy arrays

447 (if the model has multiple inputs).

448 batch_size: integer.

449 verbose: verbosity mode, 0 or 1.

450

451 Returns:

452 A numpy array of class predictions.

453 """

454 warnings.warn('`model.predict_classes()` is deprecated and '

455 'will be removed after 2021-01-01. '

456 'Please use instead:'

457 '* `np.argmax(model.predict(x), axis=-1)`, '

458 ' if your model does multi-class classification '

459 ' (e.g. if it uses a `softmax` last-layer activation).'

460 '* `(model.predict(x) > 0.5).astype("int32")`, '

461 ' if your model does binary classification '

462 ' (e.g. if it uses a `sigmoid` last-layer activation).')

463 proba = self.predict(x, batch_size=batch_size, verbose=verbose)

464 if proba.shape[-1] > 1:

465 return proba.argmax(axis=-1)

466 else:

467 return (proba > 0.5).astype('int32')

468

469 def get_config(self):

470 layer_configs = []

471 for layer in super(Sequential, self).layers:

472 # `super().layers` include the InputLayer if available (it is filtered out

473 # of `self.layers`). Note that `self._self_tracked_trackables` is managed

474 # by the tracking infrastructure and should not be used.

475 layer_configs.append(generic_utils.serialize_keras_object(layer))

476 config = {

477 'name': self.name,

478 'layers': copy.deepcopy(layer_configs)

479 }

480 if not self._is_graph_network and self._build_input_shape is not None:

481 config['build_input_shape'] = self._build_input_shape

482 return config

483

484 @classmethod

485 def from_config(cls, config, custom_objects=None):

486 if 'name' in config:

487 name = config['name']

488 build_input_shape = config.get('build_input_shape')

489 layer_configs = config['layers']

490 else:

491 name = None

492 build_input_shape = None

493 layer_configs = config

494 model = cls(name=name)

495 for layer_config in layer_configs:

496 layer = layer_module.deserialize(layer_config,

497 custom_objects=custom_objects)

498 model.add(layer)

499 if (not model.inputs and build_input_shape and

500 isinstance(build_input_shape, (tuple, list))):

501 model.build(build_input_shape)

502 return model

503

504 @property

505 def input_spec(self):

506 if hasattr(self, '_manual_input_spec'):

507 return self._manual_input_spec

508 if self.layers and hasattr(self.layers[0], 'input_spec'):

509 return self.layers[0].input_spec

510 return None

511

512 @input_spec.setter

513 def input_spec(self, value):

514 self._manual_input_spec = value

515

516 @property

517 def _trackable_saved_model_saver(self):

518 return model_serialization.SequentialSavedModelSaver(self)

519

520 def _is_layer_name_unique(self, layer):

521 for ref_layer in self.layers:

522 if layer.name == ref_layer.name and ref_layer is not layer:

523 return False

524 return True

525

526 def _assert_weights_created(self):

527 if self._graph_initialized:

528 return

529 # When the graph has not been initialized, use the Model's implementation to

530 # to check if the weights has been created.

531 super(functional.Functional, self)._assert_weights_created() # pylint: disable=bad-super-call

532

533

534def _get_shape_tuple(t):

535 if hasattr(t, 'shape'):

536 shape = t.shape

537 if isinstance(shape, tuple):

538 return shape

539 if shape.rank is not None:

540 return tuple(shape.as_list())

541 return None

542 return None

543

544

545def relax_input_shape(shape_1, shape_2):

546 if shape_1 is None or shape_2 is None:

547 return None

548 if len(shape_1) != len(shape_2):

549 return None

550 return tuple(None if d1 != d2 else d1 for d1, d2 in zip(shape_1, shape_2))

551

552

553def clear_previously_created_nodes(layer, created_nodes):

554 """Remove nodes from `created_nodes` from the layer's inbound_nodes."""

555 for node in layer._inbound_nodes:

556 prev_layers = node.inbound_layers

557 for prev_layer in nest.flatten(prev_layers):

558 prev_layer._outbound_nodes = [

559 n for n in prev_layer._outbound_nodes

560 if n not in created_nodes]

561 layer._inbound_nodes = [

562 n for n in layer._inbound_nodes if n not in created_nodes]

563

564

565def track_nodes_created_by_last_call(layer, created_nodes):

566 """Adds to `created_nodes` the nodes created by the last call to `layer`."""

567 if not layer._inbound_nodes:

568 return

569 created_nodes.add(layer._inbound_nodes[-1])

570 prev_layers = layer._inbound_nodes[-1].inbound_layers

571 for prev_layer in nest.flatten(prev_layers):

572 if prev_layer._outbound_nodes:

573 created_nodes.add(prev_layer._outbound_nodes[-1])