Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/keras/src/engine/training_eager_v1.py: 14%
113 statements
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-03 07:57 +0000
1# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""Keras training and evaluation routines for eager execution."""
17import numpy as np
18import tensorflow.compat.v2 as tf
20from keras.src import backend
21from keras.src.engine import training_utils
22from keras.src.engine import training_utils_v1
23from keras.src.mixed_precision import loss_scale_optimizer
24from keras.src.utils import losses_utils
26# isort: off
27from tensorflow.python.eager.backprop import GradientTape
28from tensorflow.python.platform import tf_logging as logging
31def _eager_loss_fn(outputs, targets, loss_fn, output_name):
32 with backend.name_scope(output_name + "_loss"):
33 loss = loss_fn(targets, outputs)
34 return loss
37def _eager_metrics_fn(model, outputs, targets, sample_weights=None, masks=None):
38 """Calculates the metrics for each output of the given model.
40 Args:
41 model: The model on which metrics are being calculated.
42 outputs: The outputs of the given model.
43 targets: The predictions or targets of the given model.
44 sample_weights: Optional list of sample weights for each output.
45 masks: Optional list of masks for each output.
47 Returns:
48 Returns the metric results for each output of the model.
49 """
50 outputs = tf.nest.flatten(outputs)
51 targets = tf.nest.flatten(targets)
52 # Invoke all(weighted and unweighted) metrics.
53 metric_results = []
54 if targets:
55 # Insert None values corresponding to the targets that need to be
56 # skipped on the model.
57 if len(model._targets) != len(targets):
58 new_targets = [
59 None if t is None else targets.pop(0) for t in model._targets
60 ]
61 targets = new_targets
63 metric_results = model._handle_metrics(
64 outputs,
65 targets=targets,
66 sample_weights=sample_weights,
67 masks=masks,
68 return_weighted_and_unweighted_metrics=True,
69 skip_target_masks=model._prepare_skip_target_masks(),
70 )
72 # Add metric results from the `add_metric` metrics.
73 metric_results.extend(
74 [
75 m.result()
76 for m in model.metrics
77 if m not in model._compile_metric_functions
78 ]
79 )
80 return metric_results
83def _model_loss(
84 model,
85 inputs,
86 targets,
87 output_loss_metrics=None,
88 sample_weights=None,
89 training=False,
90):
91 """Calculates the loss for a given model.
93 Args:
94 model: The model on which metrics are being calculated.
95 inputs: Either a dictionary of inputs to the model or a list of input
96 arrays.
97 targets: List of target arrays.
98 output_loss_metrics: List of metrics that are used to aggregated output
99 loss values.
100 sample_weights: Optional list of sample weight arrays.
101 training: Whether the model should be run in inference or training mode.
103 Returns:
104 Returns the model output, total loss, loss value calculated using the
105 specified loss function and masks for each output. The total loss
106 includes regularization losses and applies masking and sample weighting
107 to the loss value.
108 """
109 # TODO(psv): Dedup code here with graph mode prepare_total_loss() fn.
110 # Used to keep track of the total loss value (stateless).
111 # eg., total_loss = loss_weight_1 * output_1_loss_fn(...) +
112 # loss_weight_2 * output_2_loss_fn(...) +
113 # layer losses.
114 total_loss = 0
115 kwargs = {}
116 if model._expects_training_arg:
117 kwargs["training"] = training
118 if len(inputs) == 1 and not isinstance(inputs, dict):
119 inputs = inputs[0]
121 # Allow mixed `NumPy` and `EagerTensor` input here.
122 if any(
123 isinstance(input_t, (np.ndarray, float, int))
124 for input_t in tf.nest.flatten(inputs)
125 ):
126 inputs = tf.nest.map_structure(tf.convert_to_tensor, inputs)
128 outs = model(inputs, **kwargs)
129 outs = tf.nest.flatten(outs)
131 if targets:
132 targets = training_utils_v1.cast_if_floating_dtype_and_mismatch(
133 targets, outs
134 )
135 # TODO(sallymatson/psv): check if we should do same mismatch fix for weights
136 if sample_weights:
137 sample_weights = [
138 training_utils_v1.cast_if_floating_dtype(tf.convert_to_tensor(val))
139 if val is not None
140 else None
141 for val in sample_weights
142 ]
144 masks = [getattr(t, "_keras_mask", None) for t in outs]
145 targets = tf.nest.flatten(targets)
147 # Used to keep track of individual output losses.
148 output_losses = []
150 with backend.name_scope("loss"):
151 loss_fns = [
152 loss_fn for loss_fn in model.loss_functions if loss_fn is not None
153 ]
154 custom_losses = model.losses # Regularization losses
156 if not loss_fns and not custom_losses:
157 if training:
158 raise ValueError(
159 "The model cannot be trained "
160 "because it has no loss to optimize."
161 )
162 else:
163 raise ValueError(
164 "The model cannot be evaluated "
165 "because it has no loss to compute."
166 )
168 for i, loss_fn in enumerate(loss_fns):
169 weights = sample_weights[i] if sample_weights else None
170 mask = masks[i]
171 with backend.name_scope(model.output_names[i] + "_loss"):
172 if mask is not None:
173 mask = tf.cast(mask, outs[i].dtype)
174 # Update weights with mask.
175 if weights is None:
176 weights = mask
177 else:
178 # Update dimensions of weights to match with mask if
179 # possible.
180 weights = tf.cast(weights, outs[i].dtype)
181 (
182 mask,
183 _,
184 weights,
185 ) = losses_utils.squeeze_or_expand_dimensions(
186 mask, sample_weight=weights
187 )
188 weights *= mask
190 if hasattr(loss_fn, "reduction"):
191 per_sample_losses = loss_fn.call(targets[i], outs[i])
192 weighted_losses = losses_utils.compute_weighted_loss(
193 per_sample_losses,
194 sample_weight=weights,
195 reduction=losses_utils.ReductionV2.NONE,
196 )
197 loss_reduction = loss_fn.reduction
199 # `AUTO` loss reduction defaults to `SUM_OVER_BATCH_SIZE`
200 # for all compile use cases.
201 if loss_reduction == losses_utils.ReductionV2.AUTO:
202 loss_reduction = (
203 losses_utils.ReductionV2.SUM_OVER_BATCH_SIZE
204 )
206 # Compute the stateless loss value.
207 output_loss = losses_utils.reduce_weighted_loss(
208 weighted_losses, reduction=loss_reduction
209 )
210 else:
211 # Compute the stateless loss value for a custom loss class.
212 # Here we assume that the class takes care of loss reduction
213 # because if this class returns a vector value we cannot
214 # differentiate between use case where a custom optimizer
215 # expects a vector loss value vs unreduced per-sample loss
216 # value.
217 output_loss = loss_fn(
218 targets[i], outs[i], sample_weight=weights
219 )
220 loss_reduction = (
221 losses_utils.ReductionV2.SUM_OVER_BATCH_SIZE
222 )
224 # If the number of outputs is 1 then we don't append the loss metric
225 # associated with each model output. When there are multiple outputs
226 # associated with a model, each output's loss is calculated and
227 # returned as part of the loss_metrics.
228 if len(model.outputs) > 1:
229 # Keep track of the stateful output loss result.
230 output_losses.append(output_loss_metrics[i](output_loss))
232 # Scale output loss for distribution. For custom losses we assume
233 # reduction was mean.
234 if loss_reduction == losses_utils.ReductionV2.SUM_OVER_BATCH_SIZE:
235 output_loss = losses_utils.scale_loss_for_distribution(
236 output_loss
237 )
238 total_loss += model._loss_weights_list[i] * output_loss
240 # Add regularization losses
241 if custom_losses:
242 total_loss += losses_utils.scale_loss_for_distribution(
243 tf.add_n(custom_losses)
244 )
245 return outs, total_loss, output_losses, masks
248def _process_single_batch(
249 model,
250 inputs,
251 targets,
252 output_loss_metrics=None,
253 sample_weights=None,
254 training=False,
255):
256 """Calculate the loss and gradient for one input batch.
258 The model weights are updated if training is set to True.
260 Args:
261 model: Model whose loss has to be calculated.
262 inputs: List of input arrays.
263 targets: List of target arrays.
264 output_loss_metrics: List of metrics that are used to aggregated output
265 loss values.
266 sample_weights: Optional list of sample weight arrays.
267 training: The boolean represents if the weights of the model are
268 updated. 'fit' methods will set this to True while 'evaluate' methods
269 will set this to False.
271 Returns:
272 output of the model, total loss, the loss and the mask
273 associated with each output.
275 Raises:
276 ValueError: If the model has no loss to optimize.
277 """
278 with backend.eager_learning_phase_scope(
279 1 if training else 0
280 ), training_utils.RespectCompiledTrainableState(model):
281 with GradientTape() as tape:
282 outs, total_loss, output_losses, masks = _model_loss(
283 model,
284 inputs,
285 targets,
286 output_loss_metrics=output_loss_metrics,
287 sample_weights=sample_weights,
288 training=training,
289 )
290 if isinstance(
291 model.optimizer, loss_scale_optimizer.LossScaleOptimizer
292 ):
293 scaled_total_loss = model.optimizer.get_scaled_loss(total_loss)
294 else:
295 scaled_total_loss = total_loss
296 if training:
297 trainable_weights = model.trainable_weights
298 if trainable_weights:
299 # TODO(tanzheny) b/132690565: Provide mechanism for user to
300 # override model.train_on_batch.
301 if hasattr(model, "_backwards"):
302 model._backwards(tape, scaled_total_loss)
303 else:
304 grads = tape.gradient(scaled_total_loss, trainable_weights)
305 if isinstance(
306 model.optimizer, loss_scale_optimizer.LossScaleOptimizer
307 ):
308 grads = model.optimizer.get_unscaled_gradients(grads)
309 model.optimizer.apply_gradients(
310 zip(grads, trainable_weights)
311 )
312 else:
313 logging.warning(
314 "The list of trainable weights is empty. Make sure that"
315 " you are not setting model.trainable to False before "
316 "compiling the model."
317 )
318 return outs, total_loss, output_losses, masks
321def train_on_batch(
322 model, inputs, targets, sample_weights=None, output_loss_metrics=None
323):
324 """Calculates the loss and gradient updates for one input batch.
326 Args:
327 model: Model whose loss has to be calculated.
328 inputs: Input batch data.
329 targets: Target batch data.
330 sample_weights: Sample weight batch data.
331 output_loss_metrics: List of metrics that are used to aggregated output
332 loss values.
334 Returns:
335 Dict with three items:
336 'total_loss': list with a single tensor for overall loss,
337 'output_losses': list of tensors for loss corresponding to each of the
338 model output. Could be a empty list when model has only one output.
339 'metrics': list of tensors for metric specified.
340 """
341 inputs = training_utils_v1.cast_to_model_input_dtypes(inputs, model)
342 outs, total_loss, output_losses, masks = _process_single_batch(
343 model,
344 inputs,
345 targets,
346 sample_weights=sample_weights,
347 training=True,
348 output_loss_metrics=output_loss_metrics,
349 )
350 if not isinstance(outs, list):
351 outs = [outs]
352 metrics_results = _eager_metrics_fn(
353 model, outs, targets, sample_weights=sample_weights, masks=masks
354 )
355 total_loss = tf.nest.flatten(total_loss)
356 return {
357 "total_loss": total_loss,
358 "output_losses": output_losses,
359 "metrics": metrics_results,
360 }
363def test_on_batch(
364 model, inputs, targets, sample_weights=None, output_loss_metrics=None
365):
366 """Calculates the loss for one input batch.
368 Args:
369 model: Model whose loss has to be calculated.
370 inputs: Input batch data.
371 targets: Target batch data.
372 sample_weights: Sample weight batch data.
373 output_loss_metrics: List of metrics that are used to aggregated output
374 loss values.
376 Returns:
377 Dict with three items:
378 'total_loss': single tensor for overall loss,
379 'output_losses': list of tensors for loss corresponding to each of the
380 model output. Could be a empty list when model has only one output.
381 'metrics': list of tensors for metric specified.
382 """
383 inputs = training_utils_v1.cast_to_model_input_dtypes(inputs, model)
385 with backend.eager_learning_phase_scope(0):
386 outs, total_loss, output_losses, masks = _model_loss(
387 model,
388 inputs,
389 targets,
390 sample_weights=sample_weights,
391 training=False,
392 output_loss_metrics=output_loss_metrics,
393 )
394 if not isinstance(outs, list):
395 outs = [outs]
396 metrics_results = _eager_metrics_fn(
397 model, outs, targets, sample_weights=sample_weights, masks=masks
398 )
399 total_loss = tf.nest.flatten(total_loss)
401 return {
402 "total_loss": total_loss,
403 "output_losses": output_losses,
404 "metrics": metrics_results,
405 }