Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/tensorflow/python/keras/engine/training_v1.py: 20%

1# Copyright 2015 The TensorFlow Authors. All Rights Reserved. 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================== 

15"""V1 Training-related part of the Keras engine.""" 

16 

17import collections 

18import warnings 

19 

20import numpy as np 

21 

22from tensorflow.python import tf2 

23from tensorflow.python.data.ops import dataset_ops 

24from tensorflow.python.data.ops import iterator_ops 

25from tensorflow.python.distribute import distribute_lib 

26from tensorflow.python.distribute import parameter_server_strategy 

27from tensorflow.python.distribute import parameter_server_strategy_v2 

28from tensorflow.python.eager import context 

29from tensorflow.python.eager import def_function 

30from tensorflow.python.framework import constant_op 

31from tensorflow.python.framework import ops 

32from tensorflow.python.framework import sparse_tensor 

33from tensorflow.python.framework import tensor_shape 

34from tensorflow.python.framework import tensor_spec 

35from tensorflow.python.framework import tensor_util 

36from tensorflow.python.framework import type_spec 

37from tensorflow.python.keras import backend 

38from tensorflow.python.keras import losses 

39from tensorflow.python.keras import metrics as metrics_module 

40from tensorflow.python.keras import optimizer_v1 

41from tensorflow.python.keras import optimizers 

42from tensorflow.python.keras.distribute import distributed_training_utils 

43from tensorflow.python.keras.distribute import distributed_training_utils_v1 

44from tensorflow.python.keras.engine import base_layer 

45from tensorflow.python.keras.engine import training as training_lib 

46from tensorflow.python.keras.engine import training_arrays_v1 

47from tensorflow.python.keras.engine import training_distributed_v1 

48from tensorflow.python.keras.engine import training_eager_v1 

49from tensorflow.python.keras.engine import training_generator_v1 

50from tensorflow.python.keras.engine import training_utils 

51from tensorflow.python.keras.engine import training_utils_v1 

52from tensorflow.python.keras.mixed_precision import loss_scale_optimizer 

53from tensorflow.python.keras.mixed_precision import policy 

54from tensorflow.python.keras.optimizer_v2 import optimizer_v2 

55from tensorflow.python.keras.saving import saving_utils 

56from tensorflow.python.keras.saving.saved_model import model_serialization 

57from tensorflow.python.keras.utils import data_utils 

58from tensorflow.python.keras.utils import layer_utils 

59from tensorflow.python.keras.utils import losses_utils 

60from tensorflow.python.keras.utils import tf_inspect 

61from tensorflow.python.keras.utils import tf_utils 

62from tensorflow.python.keras.utils.mode_keys import ModeKeys 

63from tensorflow.python.ops import array_ops 

64from tensorflow.python.ops import math_ops 

65from tensorflow.python.platform import tf_logging as logging 

66from tensorflow.python.trackable import base as trackable 

67from tensorflow.python.types import data as data_types 

68from tensorflow.python.util import nest 

69 

70try: 

71 from scipy.sparse import issparse # pylint: disable=g-import-not-at-top 

72except ImportError: 

73 issparse = None 

74 

75 

76class Model(training_lib.Model): 

77 """`Model` groups layers into an object with training and inference features. 

78 

79 There are two ways to instantiate a `Model`: 

80 

81 1 - With the "functional API", where you start from `Input`, 

82 you chain layer calls to specify the model's forward pass, 

83 and finally you create your model from inputs and outputs: 

84 

85 ```python 

86 import tensorflow as tf 

87 

88 inputs = tf.keras.Input(shape=(3,)) 

89 x = tf.keras.layers.Dense(4, activation=tf.nn.relu)(inputs) 

90 outputs = tf.keras.layers.Dense(5, activation=tf.nn.softmax)(x) 

91 model = tf.keras.Model(inputs=inputs, outputs=outputs) 

92 ``` 

93 

94 2 - By subclassing the `Model` class: in that case, you should define your 

95 layers in `__init__` and you should implement the model's forward pass 

96 in `call`. 

97 

98 ```python 

99 import tensorflow as tf 

100 

101 class MyModel(tf.keras.Model): 

102 

103 def __init__(self): 

104 super(MyModel, self).__init__() 

105 self.dense1 = tf.keras.layers.Dense(4, activation=tf.nn.relu) 

106 self.dense2 = tf.keras.layers.Dense(5, activation=tf.nn.softmax) 

107 

108 def call(self, inputs): 

109 x = self.dense1(inputs) 

110 return self.dense2(x) 

111 

112 model = MyModel() 

113 ``` 

114 

115 If you subclass `Model`, you can optionally have 

116 a `training` argument (boolean) in `call`, which you can use to specify 

117 a different behavior in training and inference: 

118 

119 ```python 

120 import tensorflow as tf 

121 

122 class MyModel(tf.keras.Model): 

123 

124 def __init__(self): 

125 super(MyModel, self).__init__() 

126 self.dense1 = tf.keras.layers.Dense(4, activation=tf.nn.relu) 

127 self.dense2 = tf.keras.layers.Dense(5, activation=tf.nn.softmax) 

128 self.dropout = tf.keras.layers.Dropout(0.5) 

129 

130 def call(self, inputs, training=False): 

131 x = self.dense1(inputs) 

132 if training: 

133 x = self.dropout(x, training=training) 

134 return self.dense2(x) 

135 

136 model = MyModel() 

137 ``` 

138 """ 

139 

140 def __init__(self, *args, **kwargs): 

141 super(Model, self).__init__(*args, **kwargs) 

142 # initializing _distribution_strategy here since it is possible to call 

143 # predict on a model without compiling it. 

144 self._distribution_strategy = None 

145 self._compile_time_distribution_strategy = None 

146 if (ops.executing_eagerly_outside_functions() and 

147 distribute_lib.has_strategy()): 

148 self._set_strategy( 

149 distribute_lib.get_strategy()) 

150 

151 # This flag is used to track if the user is using the deprecated path of 

152 # passing distribution strategy to compile rather than creating the model 

153 # under distribution strategy scope. 

154 self._compile_distribution = False 

155 

156 self._run_eagerly = None 

157 self._experimental_run_tf_function = ( 

158 ops.executing_eagerly_outside_functions()) 

159 

160 self._v1_compile_was_called = False 

161 

162 def _init_batch_counters(self): 

163 pass # Batch counters should not be created in legacy graph mode. 

164 

165 @trackable.no_automatic_dependency_tracking 

166 def _set_strategy(self, strategy): 

167 self._compile_time_distribution_strategy = strategy 

168 

169 def get_weights(self): 

170 """Retrieves the weights of the model. 

171 

172 Returns: 

173 A flat list of Numpy arrays. 

174 """ 

175 strategy = (self._distribution_strategy or 

176 self._compile_time_distribution_strategy) 

177 if strategy: 

178 with strategy.scope(): 

179 return base_layer.Layer.get_weights(self) 

180 return base_layer.Layer.get_weights(self) 

181 

182 def load_weights(self, filepath, by_name=False, skip_mismatch=False): 

183 """Loads all layer weights, either from a TensorFlow or an HDF5 weight file. 

184 

185 If `by_name` is False weights are loaded based on the network's 

186 topology. This means the architecture should be the same as when the weights 

187 were saved. Note that layers that don't have weights are not taken into 

188 account in the topological ordering, so adding or removing layers is fine as 

189 long as they don't have weights. 

190 

191 If `by_name` is True, weights are loaded into layers only if they share the 

192 same name. This is useful for fine-tuning or transfer-learning models where 

193 some of the layers have changed. 

194 

195 Only topological loading (`by_name=False`) is supported when loading weights 

196 from the TensorFlow format. Note that topological loading differs slightly 

197 between TensorFlow and HDF5 formats for user-defined classes inheriting from 

198 `tf.keras.Model`: HDF5 loads based on a flattened list of weights, while the 

199 TensorFlow format loads based on the object-local names of attributes to 

200 which layers are assigned in the `Model`'s constructor. 

201 

202 Args: 

203 filepath: String, path to the weights file to load. For weight files in 

204 TensorFlow format, this is the file prefix (the same as was passed 

205 to `save_weights`). 

206 by_name: Boolean, whether to load weights by name or by topological 

207 order. Only topological loading is supported for weight files in 

208 TensorFlow format. 

209 skip_mismatch: Boolean, whether to skip loading of layers where there is 

210 a mismatch in the number of weights, or a mismatch in the shape of 

211 the weight (only valid when `by_name=True`). 

212 

213 Returns: 

214 When loading a weight file in TensorFlow format, returns the same status 

215 object as `tf.train.Checkpoint.restore`. When graph building, restore 

216 ops are run automatically as soon as the network is built (on first call 

217 for user-defined classes inheriting from `Model`, immediately if it is 

218 already built). 

219 

220 When loading weights in HDF5 format, returns `None`. 

221 

222 Raises: 

223 ImportError: If h5py is not available and the weight file is in HDF5 

224 format. 

225 ValueError: If `skip_mismatch` is set to `True` when `by_name` is 

226 `False`. 

227 """ 

228 if backend.is_tpu_strategy(self._distribution_strategy): 

229 if (self._distribution_strategy.extended.steps_per_run > 1 and 

230 (not saving_utils.is_hdf5_filepath(filepath))): # pylint: disable=protected-access 

231 raise ValueError('Load weights is not yet supported with TPUStrategy ' 

232 'with steps_per_run greater than 1.') 

233 return super(Model, self).load_weights(filepath, by_name, skip_mismatch) 

234 

235 @trackable.no_automatic_dependency_tracking 

236 def compile(self, 

237 optimizer='rmsprop', 

238 loss=None, 

239 metrics=None, 

240 loss_weights=None, 

241 sample_weight_mode=None, 

242 weighted_metrics=None, 

243 target_tensors=None, 

244 distribute=None, 

245 **kwargs): 

246 """Configures the model for training. 

247 

248 Args: 

249 optimizer: String (name of optimizer) or optimizer instance. 

250 See `tf.keras.optimizers`. 

251 loss: String (name of objective function), objective function or 

252 `tf.keras.losses.Loss` instance. See `tf.keras.losses`. An objective 

253 function is any callable with the signature 

254 `scalar_loss = fn(y_true, y_pred)`. If the model has multiple 

255 outputs, you can use a different loss on each output by passing a 

256 dictionary or a list of losses. The loss value that will be 

257 minimized by the model will then be the sum of all individual 

258 losses. 

259 metrics: List of metrics to be evaluated by the model during training 

260 and testing. Typically you will use `metrics=['accuracy']`. 

261 To specify different metrics for different outputs of a 

262 multi-output model, you could also pass a dictionary, such as 

263 `metrics={'output_a': 'accuracy', 'output_b': ['accuracy', 'mse']}`. 

264 You can also pass a list (len = len(outputs)) of lists of metrics 

265 such as `metrics=[['accuracy'], ['accuracy', 'mse']]` or 

266 `metrics=['accuracy', ['accuracy', 'mse']]`. 

267 loss_weights: Optional list or dictionary specifying scalar 

268 coefficients (Python floats) to weight the loss contributions 

269 of different model outputs. 

270 The loss value that will be minimized by the model 

271 will then be the *weighted sum* of all individual losses, 

272 weighted by the `loss_weights` coefficients. 

273 If a list, it is expected to have a 1:1 mapping 

274 to the model's outputs. If a tensor, it is expected to map 

275 output names (strings) to scalar coefficients. 

276 sample_weight_mode: If you need to do timestep-wise 

277 sample weighting (2D weights), set this to `"temporal"`. 

278 `None` defaults to sample-wise weights (1D). 

279 If the model has multiple outputs, you can use a different 

280 `sample_weight_mode` on each output by passing a 

281 dictionary or a list of modes. 

282 weighted_metrics: List of metrics to be evaluated and weighted 

283 by sample_weight or class_weight during training and testing. 

284 target_tensors: By default, Keras will create placeholders for the 

285 model's target, which will be fed with the target data during 

286 training. If instead you would like to use your own 

287 target tensors (in turn, Keras will not expect external 

288 Numpy data for these targets at training time), you 

289 can specify them via the `target_tensors` argument. It can be 

290 a single tensor (for a single-output model), a list of tensors, 

291 or a dict mapping output names to target tensors. 

292 distribute: NOT SUPPORTED IN TF 2.0, please create and compile the 

293 model under distribution strategy scope instead of passing it to 

294 compile. 

295 **kwargs: Any additional arguments. 

296 

297 Raises: 

298 ValueError: In case of invalid arguments for 

299 `optimizer`, `loss`, `metrics` or `sample_weight_mode`. 

300 """ 

301 self._assert_built_as_v1() 

302 self._run_eagerly = kwargs.pop('run_eagerly', None) 

303 self._experimental_run_tf_function = kwargs.pop( 

304 'experimental_run_tf_function', True) 

305 self._v1_compile_was_called = True 

306 

307 # Prepare Session arguments (legacy). 

308 kwargs.pop('cloning', None) # Legacy DistStrat argument, never used. 

309 self._from_serialized = kwargs.pop('from_serialized', False) 

310 allowed_kwargs = {'feed_dict', 'fetches', 'options', 'run_metadata'} 

311 unknown_kwargs = set(kwargs.keys()) - allowed_kwargs 

312 if unknown_kwargs: 

313 raise TypeError( 

314 'Invalid keyword argument(s) in `compile`: %s' % (unknown_kwargs,)) 

315 self._function_kwargs = kwargs 

316 if self._function_kwargs: 

317 self._experimental_run_tf_function = False 

318 if self.run_eagerly: 

319 raise ValueError( 

320 'Session keyword arguments are not supported ' 

321 'when `run_eagerly=True`. You passed the following ' 

322 'Session arguments: %s' % (self._function_kwargs,)) 

323 

324 self._set_optimizer(optimizer) 

325 is_any_keras_optimizer_v1 = any( 

326 (isinstance(opt, optimizer_v1.Optimizer) 

327 and not isinstance(opt, optimizer_v1.TFOptimizer) 

328 ) for opt in nest.flatten(self.optimizer)) 

329 

330 if is_any_keras_optimizer_v1 and ops.executing_eagerly_outside_functions(): 

331 raise ValueError('`tf.compat.v1.keras` Optimizer (', optimizer, ') is ' 

332 'not supported when eager execution is enabled. Use a ' 

333 '`tf.keras` Optimizer instead, or disable eager ' 

334 'execution.') 

335 

336 if ((target_tensors is not None) 

337 or not ops.executing_eagerly_outside_functions()): 

338 # Fallback out of things that aren't supported with v2 loops 

339 self._experimental_run_tf_function = False 

340 

341 if distribute is not None: 

342 if tf2.enabled() or self._experimental_run_tf_function: 

343 raise ValueError( 

344 'Distribute argument in compile is not available in TF 2.0 please ' 

345 'create the model under the distribution strategy scope.') 

346 logging.warning('Distribute argument in compile is deprecated please ' 

347 'create the model under the distribution strategy scope.') 

348 self._distribution_strategy = distribute 

349 self._compile_distribution = True 

350 else: 

351 if distribute_lib.has_strategy(): 

352 # When the user builds the model in the DS scope and cross replica 

353 # context we want distribution strategy to be set but when building the 

354 # replica copies of the models internally we should not be compiling 

355 # with distribution strategy and use the default compilation path. 

356 if distribute_lib.in_cross_replica_context(): 

357 self._distribution_strategy = ( 

358 distribute_lib.get_strategy()) 

359 

360 if isinstance(self._distribution_strategy, 

361 parameter_server_strategy.ParameterServerStrategyV1): 

362 raise NotImplementedError( 

363 '`tf.compat.v1.distribute.experimental.ParameterServerStrategy` ' 

364 'currently only works with the tf.Estimator API') 

365 

366 if isinstance(self._distribution_strategy, 

367 parameter_server_strategy_v2.ParameterServerStrategyV2): 

368 raise NotImplementedError( 

369 '`tf.distribute.experimental.ParameterServerStrategy` is only ' 

370 'supported in TF2.') 

371 

372 if not self._experimental_run_tf_function: 

373 self._validate_compile_param_for_distribution_strategy(self.run_eagerly, 

374 sample_weight_mode, 

375 target_tensors, 

376 weighted_metrics) 

377 # We've disabled automatic dependency tracking for this method, but do want 

378 # to add a checkpoint dependency on the optimizer if it's trackable. 

379 if isinstance(self.optimizer, trackable.Trackable): 

380 self._track_trackable( 

381 self.optimizer, name='optimizer', overwrite=True) 

382 self.loss = loss or {} 

383 self.loss_weights = loss_weights 

384 self.sample_weight_mode = sample_weight_mode 

385 self._compile_metrics = metrics or [] 

386 self._compile_weighted_metrics = weighted_metrics 

387 if self.run_eagerly and target_tensors is not None: 

388 raise ValueError( 

389 'target_tensors argument is not supported when ' 

390 'running a model eagerly.') 

391 

392 # _training_endpoints contains a list of _TrainingEndpoint object, which has 

393 # all the model output/target/loss and related metadata. 

394 self._training_endpoints = [] 

395 

396 # Used to freeze the behavior of the Model once `compile` has been called. 

397 self._compiled_trainable_state = self._get_trainable_state() 

398 

399 # Set tf.distribute.Strategy specific parameters. 

400 self._distributed_model_cache = {} 

401 self._distributed_function_cache = {} 

402 

403 # Clear any `_eager_losses` that was added. 

404 self._clear_losses() 

405 

406 if (not context.executing_eagerly() and 

407 self._distribution_strategy is not None): 

408 # Ensures a Session is created and configured correctly for Distribution 

409 # Strategy. 

410 backend.configure_and_create_distributed_session( 

411 self._distribution_strategy) 

412 # Initialize model metric attributes. 

413 self._init_metric_attributes() 

414 if not self.built or not self.inputs or not self.outputs: 

415 # Model is not compilable because it does not know its number of inputs 

416 # and outputs, nor their shapes and names. We will compile after the first 

417 # time the model gets called on training data. 

418 return 

419 self._is_compiled = True 

420 

421 # Prepare list of loss functions, same size of model outputs. 

422 self.loss_functions = training_utils_v1.prepare_loss_functions( 

423 self.loss, self.output_names) 

424 

425 target_tensors = self._process_target_tensor_for_compile(target_tensors) 

426 

427 for o, n, l, t in zip(self.outputs, self.output_names, 

428 self.loss_functions, target_tensors): 

429 endpoint = _TrainingEndpoint(o, n, l) 

430 endpoint.create_training_target(t, run_eagerly=self.run_eagerly) 

431 self._training_endpoints.append(endpoint) 

432 

433 # Prepare list loss weights, same size of model outputs. 

434 training_utils_v1.prepare_loss_weights(self._training_endpoints, 

435 loss_weights) 

436 

437 # Initialization for Eager mode execution. 

438 if self.run_eagerly: 

439 self._compile_eagerly(metrics, weighted_metrics, sample_weight_mode) 

440 return 

441 

442 with backend.get_graph().as_default(): 

443 # Save all metric attributes per output of the model. 

444 self._cache_output_metric_attributes(metrics, weighted_metrics) 

445 

446 # Set metric attributes on model. 

447 self._set_metric_attributes() 

448 

449 # Invoke metric functions (unweighted) for all the outputs. 

450 self._handle_metrics( 

451 self.outputs, 

452 targets=self._targets, 

453 skip_target_masks=self._prepare_skip_target_masks(), 

454 masks=self._prepare_output_masks()) 

455 

456 # Prepare sample weight modes. List with the same length as model outputs. 

457 training_utils_v1.prepare_sample_weight_modes( 

458 self._training_endpoints, sample_weight_mode) 

459 

460 # Creates the model loss and weighted metrics sub-graphs. 

461 self._compile_weights_loss_and_weighted_metrics() 

462 

463 # Functions for train, test and predict will 

464 # be compiled lazily when required. 

465 # This saves time when the user is not using all functions. 

466 self.train_function = None 

467 self.test_function = None 

468 self.predict_function = None 

469 

470 # Collected trainable weights, sorted in topological order. 

471 self._collected_trainable_weights = self.trainable_weights 

472 

473 # Validate all variables were correctly created in distribution scope. 

474 if self._distribution_strategy and not self._compile_distribution: 

475 for v in self.variables: 

476 strategy = self._distribution_strategy 

477 if not strategy.extended.variable_created_in_scope(v): 

478 raise ValueError( 

479 'Variable (%s) was not created in the distribution strategy ' 

480 'scope of (%s). It is most likely due to not all layers or ' 

481 'the model or optimizer being created outside the distribution ' 

482 'strategy scope. Try to make sure your code looks similar ' 

483 'to the following.\n' 

484 'with strategy.scope():\n' 

485 ' model=_create_model()\n' 

486 ' model.compile(...)'% (v, strategy)) 

487 

488 @trackable.no_automatic_dependency_tracking 

489 def _init_distributed_function_cache_if_not_compiled(self): 

490 if not hasattr(self, '_distributed_function_cache'): 

491 self._distributed_function_cache = {} 

492 

493 @property 

494 def metrics(self): 

495 """Returns the model's metrics added using `compile`, `add_metric` APIs.""" 

496 metrics = [] 

497 if self._is_compiled: 

498 if not hasattr(self, '_v1_compile_was_called'): 

499 # See b/155687393 for more details, the model is created as a v2 

500 # instance but converted to v1. Fallback to use base Model to retrieve 

501 # the metrics. 

502 return super(Model, self).metrics 

503 metrics += self._compile_metric_functions 

504 metrics.extend(self._metrics) 

505 metrics.extend( 

506 _get_metrics_from_layers( 

507 list(self._flatten_layers(include_self=False, recursive=False)))) 

508 return metrics 

509 

510 @property 

511 def metrics_names(self): 

512 """Returns the model's display labels for all outputs.""" 

513 

514 # This property includes all output names including `loss` and per-output 

515 # losses for backward compatibility. 

516 metrics_names = ['loss'] 

517 if self._is_compiled: 

518 if not hasattr(self, '_v1_compile_was_called'): 

519 # See b/155687393 for more details, the model is created as a v2 

520 # instance but converted to v1. Fallback to use base Model to retrieve 

521 # the metrics name 

522 return super(Model, self).metrics_names 

523 

524 # Add output loss metric names to the metric names list. 

525 if len(self._training_endpoints) > 1: 

526 metrics_names.extend([ 

527 e.loss_name() 

528 for e in self._training_endpoints 

529 if not e.should_skip_target() 

530 ]) 

531 

532 # Add all metric names. 

533 metrics_names += [m.name for m in self.metrics] 

534 return metrics_names 

535 

536 @property 

537 def run_eagerly(self): 

538 """Settable attribute indicating whether the model should run eagerly. 

539 

540 Running eagerly means that your model will be run step by step, 

541 like Python code. Your model might run slower, but it should become easier 

542 for you to debug it by stepping into individual layer calls. 

543 

544 By default, we will attempt to compile your model to a static graph to 

545 deliver the best execution performance. 

546 

547 Returns: 

548 Boolean, whether the model should run eagerly. 

549 """ 

550 if self._run_eagerly is True and not context.executing_eagerly(): 

551 raise ValueError('You can only set `run_eagerly=True` if eager execution ' 

552 'is enabled.') 

553 if not self.dynamic: 

554 if self._run_eagerly is None: 

555 # Respect `tf.config.run_functions_eagerly` unless 

556 # `run_eagerly` was explicitly passed to `compile`. 

557 return def_function.functions_run_eagerly() 

558 else: 

559 return self._run_eagerly 

560 else: 

561 if not context.executing_eagerly(): 

562 raise ValueError('Your model contains layers that can only be ' 

563 'successfully run in eager execution (layers ' 

564 'constructed with `dynamic=True`). ' 

565 'You must enable eager execution with ' 

566 '`tf.enable_eager_execution()`.') 

567 if self._run_eagerly is False: 

568 # TODO(fchollet): consider using py_func to enable this. 

569 raise ValueError('Your model contains layers that can only be ' 

570 'successfully run in eager execution (layers ' 

571 'constructed with `dynamic=True`). ' 

572 'You cannot set `run_eagerly=False`.') 

573 return context.executing_eagerly() 

574 

575 @run_eagerly.setter 

576 def run_eagerly(self, value): 

577 self._run_eagerly = value 

578 

579 def _select_training_loop(self, inputs): 

580 """Select training loop for fit/eval/predict based on the inputs.""" 

581 # TODO(kaftan) or TODO(scottzhu): This check should eventually be nicely 

582 # integrated into the data adapters in the v2 loop. We can't do this yet 

583 # because we currently have to fall back for unhandled data types. 

584 if isinstance(inputs, (iterator_ops.Iterator, 

585 iterator_ops.IteratorBase)): 

586 raise ValueError('For performance reasons Keras `fit`, `evaluate` and' 

587 '`predict` accept tf.data `Datasets` as input but not ' 

588 'iterators that have been manually generated from ' 

589 'Datasets by users. Please directly pass in the ' 

590 'original `Dataset` object instead of passing in ' 

591 '`iter(dataset)`.') 

592 

593 # Case 1: distribution strategy. 

594 if self._distribution_strategy: 

595 if self._in_multi_worker_mode(): 

596 return training_distributed_v1.DistributionMultiWorkerTrainingLoop( 

597 training_distributed_v1.DistributionSingleWorkerTrainingLoop()) 

598 else: 

599 return training_distributed_v1.DistributionSingleWorkerTrainingLoop() 

600 

601 # Case 2: generator-like. Input is Python generator, or Sequence object, 

602 # or a non-distributed Dataset or iterator in eager execution. 

603 if data_utils.is_generator_or_sequence(inputs): 

604 return training_generator_v1.GeneratorOrSequenceTrainingLoop() 

605 if training_utils_v1.is_eager_dataset_or_iterator(inputs): 

606 return training_generator_v1.EagerDatasetOrIteratorTrainingLoop() 

607 

608 # Case 3: Symbolic tensors or Numpy array-like. 

609 # This includes Datasets and iterators in graph mode (since they 

610 # generate symbolic tensors). 

611 if self.run_eagerly: 

612 return training_generator_v1.GeneratorLikeTrainingLoop() 

613 else: 

614 return training_arrays_v1.ArrayLikeTrainingLoop() 

615 

616 def fit(self, 

617 x=None, 

618 y=None, 

619 batch_size=None, 

620 epochs=1, 

621 verbose=1, 

622 callbacks=None, 

623 validation_split=0., 

624 validation_data=None, 

625 shuffle=True, 

626 class_weight=None, 

627 sample_weight=None, 

628 initial_epoch=0, 

629 steps_per_epoch=None, 

630 validation_steps=None, 

631 validation_freq=1, 

632 max_queue_size=10, 

633 workers=1, 

634 use_multiprocessing=False, 

635 **kwargs): 

636 """Trains the model for a fixed number of epochs (iterations on a dataset). 

637 

638 Args: 

639 x: Input data. It could be: 

640 - A Numpy array (or array-like), or a list of arrays 

641 (in case the model has multiple inputs). 

642 - A TensorFlow tensor, or a list of tensors 

643 (in case the model has multiple inputs). 

644 - A dict mapping input names to the corresponding array/tensors, 

645 if the model has named inputs. 

646 - A `tf.data` dataset. Should return a tuple 

647 of either `(inputs, targets)` or 

648 `(inputs, targets, sample_weights)`. 

649 - A generator or `keras.utils.Sequence` returning `(inputs, targets)` 

650 or `(inputs, targets, sample weights)`. 

651 y: Target data. Like the input data `x`, 

652 it could be either Numpy array(s) or TensorFlow tensor(s). 

653 It should be consistent with `x` (you cannot have Numpy inputs and 

654 tensor targets, or inversely). If `x` is a dataset, generator, 

655 or `keras.utils.Sequence` instance, `y` should 

656 not be specified (since targets will be obtained from `x`). 

657 batch_size: Integer or `None`. 

658 Number of samples per gradient update. 

659 If unspecified, `batch_size` will default to 32. 

660 Do not specify the `batch_size` if your data is in the 

661 form of symbolic tensors, datasets, 

662 generators, or `keras.utils.Sequence` instances (since they generate 

663 batches). 

664 epochs: Integer. Number of epochs to train the model. 

665 An epoch is an iteration over the entire `x` and `y` 

666 data provided. 

667 Note that in conjunction with `initial_epoch`, 

668 `epochs` is to be understood as "final epoch". 

669 The model is not trained for a number of iterations 

670 given by `epochs`, but merely until the epoch 

671 of index `epochs` is reached. 

672 verbose: 0, 1, or 2. Verbosity mode. 

673 0 = silent, 1 = progress bar, 2 = one line per epoch. 

674 Note that the progress bar is not particularly useful when 

675 logged to a file, so verbose=2 is recommended when not running 

676 interactively (eg, in a production environment). 

677 callbacks: List of `keras.callbacks.Callback` instances. 

678 List of callbacks to apply during training. 

679 See `tf.keras.callbacks`. 

680 validation_split: Float between 0 and 1. 

681 Fraction of the training data to be used as validation data. 

682 The model will set apart this fraction of the training data, 

683 will not train on it, and will evaluate 

684 the loss and any model metrics 

685 on this data at the end of each epoch. 

686 The validation data is selected from the last samples 

687 in the `x` and `y` data provided, before shuffling. This argument is 

688 not supported when `x` is a dataset, generator or 

689 `keras.utils.Sequence` instance. 

690 validation_data: Data on which to evaluate 

691 the loss and any model metrics at the end of each epoch. 

692 The model will not be trained on this data. 

693 `validation_data` will override `validation_split`. 

694 `validation_data` could be: 

695 - tuple `(x_val, y_val)` of Numpy arrays or tensors 

696 - tuple `(x_val, y_val, val_sample_weights)` of Numpy arrays 

697 - dataset 

698 For the first two cases, `batch_size` must be provided. 

699 For the last case, `validation_steps` could be provided. 

700 shuffle: Boolean (whether to shuffle the training data 

701 before each epoch) or str (for 'batch'). 

702 'batch' is a special option for dealing with the 

703 limitations of HDF5 data; it shuffles in batch-sized chunks. 

704 Has no effect when `steps_per_epoch` is not `None`. 

705 class_weight: Optional dictionary mapping class indices (integers) 

706 to a weight (float) value, used for weighting the loss function 

707 (during training only). 

708 This can be useful to tell the model to 

709 "pay more attention" to samples from 

710 an under-represented class. 

711 sample_weight: Optional Numpy array of weights for 

712 the training samples, used for weighting the loss function 

713 (during training only). You can either pass a flat (1D) 

714 Numpy array with the same length as the input samples 

715 (1:1 mapping between weights and samples), 

716 or in the case of temporal data, 

717 you can pass a 2D array with shape 

718 `(samples, sequence_length)`, 

719 to apply a different weight to every timestep of every sample. 

720 In this case you should make sure to specify 

721 `sample_weight_mode="temporal"` in `compile()`. This argument is not 

722 supported when `x` is a dataset, generator, or 

723 `keras.utils.Sequence` instance, instead provide the sample_weights 

724 as the third element of `x`. 

725 initial_epoch: Integer. 

726 Epoch at which to start training 

727 (useful for resuming a previous training run). 

728 steps_per_epoch: Integer or `None`. 

729 Total number of steps (batches of samples) 

730 before declaring one epoch finished and starting the 

731 next epoch. When training with input tensors such as 

732 TensorFlow data tensors, the default `None` is equal to 

733 the number of samples in your dataset divided by 

734 the batch size, or 1 if that cannot be determined. If x is a 

735 `tf.data` dataset, and 'steps_per_epoch' 

736 is None, the epoch will run until the input dataset is exhausted. 

737 This argument is not supported with array inputs. 

738 validation_steps: Only relevant if `validation_data` is provided and 

739 is a `tf.data` dataset. Total number of steps (batches of 

740 samples) to draw before stopping when performing validation 

741 at the end of every epoch. If 'validation_steps' is None, validation 

742 will run until the `validation_data` dataset is exhausted. In the 

743 case of a infinite dataset, it will run into a infinite loop. 

744 If 'validation_steps' is specified and only part of the dataset 

745 will be consumed, the evaluation will start from the beginning of 

746 the dataset at each epoch. This ensures that the same validation 

747 samples are used every time. 

748 validation_freq: Only relevant if validation data is provided. Integer 

749 or `collections.abc.Container` instance (e.g. list, tuple, etc.). 

750 If an integer, specifies how many training epochs to run before a 

751 new validation run is performed, e.g. `validation_freq=2` runs 

752 validation every 2 epochs. If a Container, specifies the epochs on 

753 which to run validation, e.g. `validation_freq=[1, 2, 10]` runs 

754 validation at the end of the 1st, 2nd, and 10th epochs. 

755 max_queue_size: Integer. Used for generator or `keras.utils.Sequence` 

756 input only. Maximum size for the generator queue. 

757 If unspecified, `max_queue_size` will default to 10. 

758 workers: Integer. Used for generator or `keras.utils.Sequence` input 

759 only. Maximum number of processes to spin up 

760 when using process-based threading. If unspecified, `workers` 

761 will default to 1. If 0, will execute the generator on the main 

762 thread. 

763 use_multiprocessing: Boolean. Used for generator or 

764 `keras.utils.Sequence` input only. If `True`, use process-based 

765 threading. If unspecified, `use_multiprocessing` will default to 

766 `False`. Note that because this implementation relies on 

767 multiprocessing, you should not pass non-picklable arguments to 

768 the generator as they can't be passed easily to children processes. 

769 **kwargs: Used for backwards compatibility. 

770 

771 Returns: 

772 A `History` object. Its `History.history` attribute is 

773 a record of training loss values and metrics values 

774 at successive epochs, as well as validation loss values 

775 and validation metrics values (if applicable). 

776 

777 Raises: 

778 RuntimeError: If the model was never compiled. 

779 ValueError: In case of mismatch between the provided input data 

780 and what the model expects. 

781 """ 

782 self._assert_built_as_v1() 

783 # Legacy support 

784 if 'nb_epoch' in kwargs: 

785 logging.warning( 

786 'The `nb_epoch` argument in `fit` has been renamed `epochs`.') 

787 epochs = kwargs.pop('nb_epoch') 

788 if kwargs: 

789 raise TypeError('Unrecognized keyword arguments: ' + str(kwargs)) 

790 self._assert_compile_was_called() 

791 self._check_call_args('fit') 

792 

793 func = self._select_training_loop(x) 

794 return func.fit( 

795 self, 

796 x=x, 

797 y=y, 

798 batch_size=batch_size, 

799 epochs=epochs, 

800 verbose=verbose, 

801 callbacks=callbacks, 

802 validation_split=validation_split, 

803 validation_data=validation_data, 

804 shuffle=shuffle, 

805 class_weight=class_weight, 

806 sample_weight=sample_weight, 

807 initial_epoch=initial_epoch, 

808 steps_per_epoch=steps_per_epoch, 

809 validation_steps=validation_steps, 

810 validation_freq=validation_freq, 

811 max_queue_size=max_queue_size, 

812 workers=workers, 

813 use_multiprocessing=use_multiprocessing) 

814 

815 def evaluate(self, 

816 x=None, 

817 y=None, 

818 batch_size=None, 

819 verbose=1, 

820 sample_weight=None, 

821 steps=None, 

822 callbacks=None, 

823 max_queue_size=10, 

824 workers=1, 

825 use_multiprocessing=False): 

826 """Returns the loss value & metrics values for the model in test mode. 

827 

828 Computation is done in batches (see the `batch_size` arg.) 

829 

830 Args: 

831 x: Input data. It could be: 

832 - A Numpy array (or array-like), or a list of arrays 

833 (in case the model has multiple inputs). 

834 - A TensorFlow tensor, or a list of tensors 

835 (in case the model has multiple inputs). 

836 - A dict mapping input names to the corresponding array/tensors, 

837 if the model has named inputs. 

838 - A `tf.data` dataset. 

839 - A generator or `keras.utils.Sequence` instance. 

840 y: Target data. Like the input data `x`, 

841 it could be either Numpy array(s) or TensorFlow tensor(s). 

842 It should be consistent with `x` (you cannot have Numpy inputs and 

843 tensor targets, or inversely). 

844 If `x` is a dataset, generator or 

845 `keras.utils.Sequence` instance, `y` should not be specified (since 

846 targets will be obtained from the iterator/dataset). 

847 batch_size: Integer or `None`. 

848 Number of samples per batch of computation. 

849 If unspecified, `batch_size` will default to 32. 

850 Do not specify the `batch_size` if your data is in the 

851 form of symbolic tensors, dataset, 

852 generators, or `keras.utils.Sequence` instances (since they generate 

853 batches). 

854 verbose: 0 or 1. Verbosity mode. 

855 0 = silent, 1 = progress bar. 

856 sample_weight: Optional Numpy array of weights for 

857 the test samples, used for weighting the loss function. 

858 You can either pass a flat (1D) 

859 Numpy array with the same length as the input samples 

860 (1:1 mapping between weights and samples), 

861 or in the case of temporal data, 

862 you can pass a 2D array with shape 

863 `(samples, sequence_length)`, 

864 to apply a different weight to every timestep of every sample. 

865 In this case you should make sure to specify 

866 `sample_weight_mode="temporal"` in `compile()`. This argument is not 

867 supported when `x` is a dataset, instead pass 

868 sample weights as the third element of `x`. 

869 steps: Integer or `None`. 

870 Total number of steps (batches of samples) 

871 before declaring the evaluation round finished. 

872 Ignored with the default value of `None`. 

873 If x is a `tf.data` dataset and `steps` is 

874 None, 'evaluate' will run until the dataset is exhausted. 

875 This argument is not supported with array inputs. 

876 callbacks: List of `keras.callbacks.Callback` instances. 

877 List of callbacks to apply during evaluation. 

878 See [callbacks](/api_docs/python/tf/keras/callbacks). 

879 max_queue_size: Integer. Used for generator or `keras.utils.Sequence` 

880 input only. Maximum size for the generator queue. 

881 If unspecified, `max_queue_size` will default to 10. 

882 workers: Integer. Used for generator or `keras.utils.Sequence` input 

883 only. Maximum number of processes to spin up when using 

884 process-based threading. If unspecified, `workers` will default 

885 to 1. If 0, will execute the generator on the main thread. 

886 use_multiprocessing: Boolean. Used for generator or 

887 `keras.utils.Sequence` input only. If `True`, use process-based 

888 threading. If unspecified, `use_multiprocessing` will default to 

889 `False`. Note that because this implementation relies on 

890 multiprocessing, you should not pass non-picklable arguments to 

891 the generator as they can't be passed easily to children processes. 

892 

893 Returns: 

894 Scalar test loss (if the model has a single output and no metrics) 

895 or list of scalars (if the model has multiple outputs 

896 and/or metrics). The attribute `model.metrics_names` will give you 

897 the display labels for the scalar outputs. 

898 

899 Raises: 

900 ValueError: in case of invalid arguments. 

901 """ 

902 self._assert_built_as_v1() 

903 self._assert_compile_was_called() 

904 self._check_call_args('evaluate') 

905 

906 func = self._select_training_loop(x) 

907 return func.evaluate( 

908 self, 

909 x=x, 

910 y=y, 

911 batch_size=batch_size, 

912 verbose=verbose, 

913 sample_weight=sample_weight, 

914 steps=steps, 

915 callbacks=callbacks, 

916 max_queue_size=max_queue_size, 

917 workers=workers, 

918 use_multiprocessing=use_multiprocessing) 

919 

920 def predict(self, 

921 x, 

922 batch_size=None, 

923 verbose=0, 

924 steps=None, 

925 callbacks=None, 

926 max_queue_size=10, 

927 workers=1, 

928 use_multiprocessing=False): 

929 """Generates output predictions for the input samples. 

930 

931 Computation is done in batches (see the `batch_size` arg.) 

932 

933 Args: 

934 x: Input samples. It could be: 

935 - A Numpy array (or array-like), or a list of arrays 

936 (in case the model has multiple inputs). 

937 - A TensorFlow tensor, or a list of tensors 

938 (in case the model has multiple inputs). 

939 - A `tf.data` dataset. 

940 - A generator or `keras.utils.Sequence` instance. 

941 batch_size: Integer or `None`. 

942 Number of samples per batch of computation. 

943 If unspecified, `batch_size` will default to 32. 

944 Do not specify the `batch_size` if your data is in the 

945 form of symbolic tensors, dataset, 

946 generators, or `keras.utils.Sequence` instances (since they generate 

947 batches). 

948 verbose: Verbosity mode, 0 or 1. 

949 steps: Total number of steps (batches of samples) 

950 before declaring the prediction round finished. 

951 Ignored with the default value of `None`. If x is a `tf.data` 

952 dataset and `steps` is None, `predict` will 

953 run until the input dataset is exhausted. 

954 callbacks: List of `keras.callbacks.Callback` instances. 

955 List of callbacks to apply during prediction. 

956 See [callbacks](/api_docs/python/tf/keras/callbacks). 

957 max_queue_size: Integer. Used for generator or `keras.utils.Sequence` 

958 input only. Maximum size for the generator queue. 

959 If unspecified, `max_queue_size` will default to 10. 

960 workers: Integer. Used for generator or `keras.utils.Sequence` input 

961 only. Maximum number of processes to spin up when using 

962 process-based threading. If unspecified, `workers` will default 

963 to 1. If 0, will execute the generator on the main thread. 

964 use_multiprocessing: Boolean. Used for generator or 

965 `keras.utils.Sequence` input only. If `True`, use process-based 

966 threading. If unspecified, `use_multiprocessing` will default to 

967 `False`. Note that because this implementation relies on 

968 multiprocessing, you should not pass non-picklable arguments to 

969 the generator as they can't be passed easily to children processes. 

970 

971 

972 Returns: 

973 Numpy array(s) of predictions. 

974 

975 Raises: 

976 ValueError: In case of mismatch between the provided 

977 input data and the model's expectations, 

978 or in case a stateful model receives a number of samples 

979 that is not a multiple of the batch size. 

980 """ 

981 self._assert_built_as_v1() 

982 self._check_call_args('predict') 

983 

984 func = self._select_training_loop(x) 

985 return func.predict( 

986 self, 

987 x=x, 

988 batch_size=batch_size, 

989 verbose=verbose, 

990 steps=steps, 

991 callbacks=callbacks, 

992 max_queue_size=max_queue_size, 

993 workers=workers, 

994 use_multiprocessing=use_multiprocessing) 

995 

996 def reset_metrics(self): 

997 """Resets the state of metrics.""" 

998 metrics = self._get_training_eval_metrics() 

999 for m in metrics: 

1000 m.reset_state() 

1001 

1002 # Reset metrics on all the distributed (cloned) models. 

1003 if self._distribution_strategy: 

1004 distributed_training_utils_v1._reset_metrics(self) # pylint: disable=protected-access 

1005 

1006 def train_on_batch(self, 

1007 x, 

1008 y=None, 

1009 sample_weight=None, 

1010 class_weight=None, 

1011 reset_metrics=True): 

1012 """Runs a single gradient update on a single batch of data. 

1013 

1014 Args: 

1015 x: Input data. It could be: 

1016 - A Numpy array (or array-like), or a list of arrays 

1017 (in case the model has multiple inputs). 

1018 - A TensorFlow tensor, or a list of tensors 

1019 (in case the model has multiple inputs). 

1020 - A dict mapping input names to the corresponding array/tensors, 

1021 if the model has named inputs. 

1022 - A `tf.data` dataset. 

1023 y: Target data. Like the input data `x`, it could be either Numpy 

1024 array(s) or TensorFlow tensor(s). It should be consistent with `x` 

1025 (you cannot have Numpy inputs and tensor targets, or inversely). If