Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/keras/src/saving/legacy/hdf5_format.py: 8%
405 statements
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-03 07:57 +0000
1# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
16"""Functions for saving and loading a Keras Model from HDF5 format."""
18import json
19import os
21import numpy as np
22import tensorflow.compat.v2 as tf
24from keras.src import backend
25from keras.src.optimizers import optimizer as optimizer_base
26from keras.src.optimizers import optimizer_v1
27from keras.src.saving import object_registration
28from keras.src.saving.legacy import model_config as model_config_lib
29from keras.src.saving.legacy import saving_utils
30from keras.src.saving.legacy.saved_model import json_utils
31from keras.src.utils.generic_utils import LazyLoader
32from keras.src.utils.io_utils import ask_to_proceed_with_overwrite
34# isort: off
35from tensorflow.python.platform import tf_logging as logging
37try:
38 import h5py
40 HDF5_OBJECT_HEADER_LIMIT = 64512
41except ImportError:
42 h5py = None
44# TODO(b/134426265): Switch back to single-quotes to match the rest of the file
45# once the issue with copybara is fixed.
47sequential_lib = LazyLoader(
48 "sequential_lib", globals(), "keras.src.engine.sequential"
49)
52def save_model_to_hdf5(model, filepath, overwrite=True, include_optimizer=True):
53 """Saves a model to a HDF5 file.
55 The saved model contains:
56 - the model's configuration (topology)
57 - the model's weights
58 - the model's optimizer's state (if any)
60 Thus the saved model can be reinstantiated in
61 the exact same state, without any of the code
62 used for model definition or training.
64 Args:
65 model: Keras model instance to be saved.
66 filepath: One of the following:
67 - String, path where to save the model
68 - `h5py.File` object where to save the model
69 overwrite: Whether we should overwrite any existing
70 model at the target location, or instead
71 ask the user with a manual prompt.
72 include_optimizer: If True, save optimizer's state together.
74 Raises:
75 ImportError: if h5py is not available.
76 """
78 if h5py is None:
79 raise ImportError(
80 "`save_model()` using h5 format requires h5py. Could not "
81 "import h5py."
82 )
84 # Ensures that all models saved in HDF5 format follow the old serialization
85 model.use_legacy_config = True
87 # TODO(psv) Add warning when we save models that contain non-serializable
88 # entities like metrics added using `add_metric` and losses added using
89 # `add_loss.`
90 if len(model.weights) != len(model._undeduplicated_weights):
91 logging.warning(
92 "Found duplicated `Variable`s in Model's `weights`. "
93 "This is usually caused by `Variable`s being shared by "
94 "Layers in the Model. These `Variable`s will be treated "
95 "as separate `Variable`s when the Model is restored. To "
96 'avoid this, please save with `save_format="tf"`.'
97 )
99 if not isinstance(filepath, h5py.File):
100 # If file exists and should not be overwritten.
101 if not overwrite and os.path.isfile(filepath):
102 proceed = ask_to_proceed_with_overwrite(filepath)
103 if not proceed:
104 return
106 # Try creating dir if not exist
107 dirpath = os.path.dirname(filepath)
108 if not os.path.exists(dirpath):
109 tf.io.gfile.makedirs(dirpath)
111 f = h5py.File(filepath, mode="w")
112 opened_new_file = True
113 else:
114 f = filepath
115 opened_new_file = False
117 try:
118 model_metadata = saving_utils.model_metadata(model, include_optimizer)
119 for k, v in model_metadata.items():
120 if isinstance(v, (dict, list, tuple)):
121 f.attrs[k] = json.dumps(
122 v, default=json_utils.get_json_type
123 ).encode("utf8")
124 else:
125 f.attrs[k] = v
127 model_weights_group = f.create_group("model_weights")
128 save_weights_to_hdf5_group(model_weights_group, model)
130 # TODO(b/128683857): Add integration tests between tf.keras and external
131 # Keras, to avoid breaking TF.js users.
132 if (
133 include_optimizer
134 and model.optimizer
135 and not isinstance(model.optimizer, optimizer_v1.TFOptimizer)
136 ):
137 save_optimizer_weights_to_hdf5_group(f, model.optimizer)
139 f.flush()
140 finally:
141 if opened_new_file:
142 f.close()
145def load_model_from_hdf5(filepath, custom_objects=None, compile=True):
146 """Loads a model saved via `save_model_to_hdf5`.
148 Args:
149 filepath: One of the following:
150 - String, path to the saved model
151 - `h5py.File` object from which to load the model
152 custom_objects: Optional dictionary mapping names
153 (strings) to custom classes or functions to be
154 considered during deserialization.
155 compile: Boolean, whether to compile the model
156 after loading.
158 Returns:
159 A Keras model instance. If an optimizer was found
160 as part of the saved model, the model is already
161 compiled. Otherwise, the model is uncompiled and
162 a warning will be displayed. When `compile` is set
163 to False, the compilation is omitted without any
164 warning.
166 Raises:
167 ImportError: if h5py is not available.
168 ValueError: In case of an invalid savefile.
169 """
170 if h5py is None:
171 raise ImportError(
172 "`load_model()` using h5 format requires h5py. Could not "
173 "import h5py."
174 )
176 if not custom_objects:
177 custom_objects = {}
179 tlco = object_registration._THREAD_LOCAL_CUSTOM_OBJECTS.__dict__
180 gco = object_registration._GLOBAL_CUSTOM_OBJECTS
181 custom_objects = {**custom_objects, **tlco, **gco}
183 opened_new_file = not isinstance(filepath, h5py.File)
184 if opened_new_file:
185 f = h5py.File(filepath, mode="r")
186 else:
187 f = filepath
189 model = None
190 try:
191 # instantiate model
192 model_config = f.attrs.get("model_config")
193 if model_config is None:
194 raise ValueError(
195 f"No model config found in the file at {filepath}."
196 )
197 if hasattr(model_config, "decode"):
198 model_config = model_config.decode("utf-8")
199 model_config = json_utils.decode(model_config)
200 model = model_config_lib.model_from_config(
201 model_config, custom_objects=custom_objects
202 )
204 # set weights
205 load_weights_from_hdf5_group(f["model_weights"], model)
207 if compile:
208 # instantiate optimizer
209 training_config = f.attrs.get("training_config")
210 if hasattr(training_config, "decode"):
211 training_config = training_config.decode("utf-8")
212 if training_config is None:
213 logging.warning(
214 "No training configuration found in the save file, so "
215 "the model was *not* compiled. Compile it manually."
216 )
217 return model
218 training_config = json_utils.decode(training_config)
220 # Compile model.
221 model.compile(
222 **saving_utils.compile_args_from_training_config(
223 training_config, custom_objects
224 ),
225 from_serialized=True,
226 )
227 saving_utils.try_build_compiled_arguments(model)
229 # Set optimizer weights.
230 if "optimizer_weights" in f:
231 try:
232 if isinstance(model.optimizer, optimizer_base.Optimizer):
233 model.optimizer.build(model.trainable_variables)
234 else:
235 model.optimizer._create_all_weights(
236 model.trainable_variables
237 )
238 except (NotImplementedError, AttributeError):
239 logging.warning(
240 "Error when creating the weights of optimizer {}, "
241 "making it impossible to restore the saved optimizer "
242 "state. As a result, your model is starting with "
243 "a freshly initialized optimizer."
244 )
246 optimizer_weight_values = (
247 load_optimizer_weights_from_hdf5_group(f)
248 )
249 try:
250 model.optimizer.set_weights(optimizer_weight_values)
251 except ValueError:
252 logging.warning(
253 "Error in loading the saved optimizer "
254 "state. As a result, your model is "
255 "starting with a freshly initialized "
256 "optimizer."
257 )
258 finally:
259 if opened_new_file:
260 f.close()
261 return model
264def preprocess_weights_for_loading(
265 layer, weights, original_keras_version=None, original_backend=None
266):
267 """Preprocess layer weights between different Keras formats.
269 Converts layers weights from Keras 1 format to Keras 2 and also weights of
270 cuDNN layers in Keras 2.
272 Args:
273 layer: Layer instance.
274 weights: List of weights values (Numpy arrays).
275 original_keras_version: Keras version for the weights, as a string.
276 original_backend: Keras backend the weights were trained with,
277 as a string.
279 Returns:
280 A list of weights values (Numpy arrays).
281 """
283 def convert_nested_bidirectional(weights):
284 """Converts layers nested in `Bidirectional` wrapper.
286 This function uses `preprocess_weights_for_loading()` for converting
287 layers.
289 Args:
290 weights: List of weights values (Numpy arrays).
292 Returns:
293 A list of weights values (Numpy arrays).
294 """
295 num_weights_per_layer = len(weights) // 2
296 forward_weights = preprocess_weights_for_loading(
297 layer.forward_layer,
298 weights[:num_weights_per_layer],
299 original_keras_version,
300 original_backend,
301 )
302 backward_weights = preprocess_weights_for_loading(
303 layer.backward_layer,
304 weights[num_weights_per_layer:],
305 original_keras_version,
306 original_backend,
307 )
308 return forward_weights + backward_weights
310 def convert_nested_time_distributed(weights):
311 """Converts layers nested in `TimeDistributed` wrapper.
313 This function uses `preprocess_weights_for_loading()` for converting
314 nested layers.
316 Args:
317 weights: List of weights values (Numpy arrays).
319 Returns:
320 A list of weights values (Numpy arrays).
321 """
322 return preprocess_weights_for_loading(
323 layer.layer, weights, original_keras_version, original_backend
324 )
326 def convert_nested_model(weights):
327 """Converts layers nested in `Model` or `Sequential`.
329 This function uses `preprocess_weights_for_loading()` for converting
330 nested layers.
332 Args:
333 weights: List of weights values (Numpy arrays).
335 Returns:
336 A list of weights values (Numpy arrays).
337 """
338 trainable_weights = weights[: len(layer.trainable_weights)]
339 non_trainable_weights = weights[len(layer.trainable_weights) :]
341 new_trainable_weights = []
342 new_non_trainable_weights = []
344 for sublayer in layer.layers:
345 num_trainable_weights = len(sublayer.trainable_weights)
346 num_non_trainable_weights = len(sublayer.non_trainable_weights)
347 if sublayer.weights:
348 preprocessed = preprocess_weights_for_loading(
349 layer=sublayer,
350 weights=(
351 trainable_weights[:num_trainable_weights]
352 + non_trainable_weights[:num_non_trainable_weights]
353 ),
354 original_keras_version=original_keras_version,
355 original_backend=original_backend,
356 )
357 new_trainable_weights.extend(
358 preprocessed[:num_trainable_weights]
359 )
360 new_non_trainable_weights.extend(
361 preprocessed[num_trainable_weights:]
362 )
364 trainable_weights = trainable_weights[num_trainable_weights:]
365 non_trainable_weights = non_trainable_weights[
366 num_non_trainable_weights:
367 ]
368 new_trainable_weights += layer._trainable_weights
369 new_non_trainable_weights += layer._non_trainable_weights
370 return new_trainable_weights + new_non_trainable_weights
372 # Convert layers nested in Bidirectional/Model/Sequential.
373 # Both transformation should be ran for both Keras 1->2 conversion
374 # and for conversion of cuDNN layers.
375 if layer.__class__.__name__ == "Bidirectional":
376 weights = convert_nested_bidirectional(weights)
377 if layer.__class__.__name__ == "TimeDistributed":
378 weights = convert_nested_time_distributed(weights)
379 elif layer.__class__.__name__ in ["Model", "Sequential", "Functional"]:
380 weights = convert_nested_model(weights)
382 if original_keras_version == "1":
383 if layer.__class__.__name__ == "TimeDistributed":
384 weights = preprocess_weights_for_loading(
385 layer.layer, weights, original_keras_version, original_backend
386 )
388 if layer.__class__.__name__ == "Conv1D":
389 shape = weights[0].shape
390 # Handle Keras 1.1 format
391 if (
392 shape[:2] != (layer.kernel_size[0], 1)
393 or shape[3] != layer.filters
394 ):
395 # Legacy shape:
396 # (filters, input_dim, filter_length, 1)
397 assert shape[0] == layer.filters and shape[2:] == (
398 layer.kernel_size[0],
399 1,
400 )
401 weights[0] = np.transpose(weights[0], (2, 3, 1, 0))
402 weights[0] = weights[0][:, 0, :, :]
404 if layer.__class__.__name__ == "Conv2D":
405 if layer.data_format == "channels_first":
406 # old: (filters, stack_size, kernel_rows, kernel_cols)
407 # new: (kernel_rows, kernel_cols, stack_size, filters)
408 weights[0] = np.transpose(weights[0], (2, 3, 1, 0))
410 if layer.__class__.__name__ == "Conv2DTranspose":
411 if layer.data_format == "channels_last":
412 # old: (kernel_rows, kernel_cols, stack_size, filters)
413 # new: (kernel_rows, kernel_cols, filters, stack_size)
414 weights[0] = np.transpose(weights[0], (0, 1, 3, 2))
415 if layer.data_format == "channels_first":
416 # old: (filters, stack_size, kernel_rows, kernel_cols)
417 # new: (kernel_rows, kernel_cols, filters, stack_size)
418 weights[0] = np.transpose(weights[0], (2, 3, 0, 1))
420 if layer.__class__.__name__ == "Conv3D":
421 if layer.data_format == "channels_first":
422 # old: (filters, stack_size, ...)
423 # new: (..., stack_size, filters)
424 weights[0] = np.transpose(weights[0], (2, 3, 4, 1, 0))
426 if layer.__class__.__name__ == "GRU":
427 if len(weights) == 9:
428 kernel = np.concatenate(
429 [weights[0], weights[3], weights[6]], axis=-1
430 )
431 recurrent_kernel = np.concatenate(
432 [weights[1], weights[4], weights[7]], axis=-1
433 )
434 bias = np.concatenate(
435 [weights[2], weights[5], weights[8]], axis=-1
436 )
437 weights = [kernel, recurrent_kernel, bias]
439 if layer.__class__.__name__ == "LSTM":
440 if len(weights) == 12:
441 # old: i, c, f, o
442 # new: i, f, c, o
443 kernel = np.concatenate(
444 [weights[0], weights[6], weights[3], weights[9]], axis=-1
445 )
446 recurrent_kernel = np.concatenate(
447 [weights[1], weights[7], weights[4], weights[10]], axis=-1
448 )
449 bias = np.concatenate(
450 [weights[2], weights[8], weights[5], weights[11]], axis=-1
451 )
452 weights = [kernel, recurrent_kernel, bias]
454 if layer.__class__.__name__ == "ConvLSTM2D":
455 if len(weights) == 12:
456 kernel = np.concatenate(
457 [weights[0], weights[6], weights[3], weights[9]], axis=-1
458 )
459 recurrent_kernel = np.concatenate(
460 [weights[1], weights[7], weights[4], weights[10]], axis=-1
461 )
462 bias = np.concatenate(
463 [weights[2], weights[8], weights[5], weights[11]], axis=-1
464 )
465 if layer.data_format == "channels_first":
466 # old: (filters, stack_size, kernel_rows, kernel_cols)
467 # new: (kernel_rows, kernel_cols, stack_size, filters)
468 kernel = np.transpose(kernel, (2, 3, 1, 0))
469 recurrent_kernel = np.transpose(
470 recurrent_kernel, (2, 3, 1, 0)
471 )
472 weights = [kernel, recurrent_kernel, bias]
474 conv_layers = [
475 "Conv1D",
476 "Conv2D",
477 "Conv3D",
478 "Conv2DTranspose",
479 "ConvLSTM2D",
480 ]
481 if layer.__class__.__name__ in conv_layers:
482 if backend.int_shape(layer.weights[0]) != weights[0].shape:
483 weights[0] = np.transpose(weights[0], (3, 2, 0, 1))
484 if layer.__class__.__name__ == "ConvLSTM2D":
485 weights[1] = np.transpose(weights[1], (3, 2, 0, 1))
487 # convert cuDNN layers
488 return _convert_rnn_weights(layer, weights)
491def _convert_rnn_weights(layer, weights):
492 """Converts weights for RNN layers between native and cuDNN format.
494 Input kernels for each gate are transposed and converted between Fortran
495 and C layout, recurrent kernels are transposed. For LSTM biases are summed/
496 split in half, for GRU biases are reshaped.
498 Weights can be converted in both directions between `LSTM` and`CuDNNSLTM`
499 and between `CuDNNGRU` and `GRU(reset_after=True)`. Default `GRU` is not
500 compatible with `CuDNNGRU`.
502 For missing biases in `LSTM`/`GRU` (`use_bias=False`) no conversion is made.
504 Args:
505 layer: Target layer instance.
506 weights: List of source weights values (input kernels, recurrent
507 kernels, [biases]) (Numpy arrays).
509 Returns:
510 A list of converted weights values (Numpy arrays).
512 Raises:
513 ValueError: for incompatible GRU layer/weights or incompatible biases
514 """
516 def transform_kernels(kernels, func, n_gates):
517 """Transforms kernel for each gate separately using given function.
519 Args:
520 kernels: Stacked array of kernels for individual gates.
521 func: Function applied to kernel of each gate.
522 n_gates: Number of gates (4 for LSTM, 3 for GRU).
524 Returns:
525 Stacked array of transformed kernels.
526 """
527 return np.hstack([func(k) for k in np.hsplit(kernels, n_gates)])
529 def transpose_input(from_cudnn):
530 """Makes a function that transforms input kernels from/to cuDNN format.
532 It keeps the shape, but changes between the layout (Fortran/C). Eg.:
534 ```
535 Keras cuDNN
536 [[0, 1, 2], <---> [[0, 2, 4],
537 [3, 4, 5]] [1, 3, 5]]
538 ```
540 It can be passed to `transform_kernels()`.
542 Args:
543 from_cudnn: `True` if source weights are in cuDNN format, `False` if
544 they're in plain Keras format.
546 Returns:
547 Function that converts input kernel to the other format.
548 """
549 order = "F" if from_cudnn else "C"
551 def transform(kernel):
552 return kernel.T.reshape(kernel.shape, order=order)
554 return transform
556 target_class = layer.__class__.__name__
558 # convert the weights between CuDNNLSTM and LSTM
559 if target_class in ["LSTM", "CuDNNLSTM"] and len(weights) == 3:
560 # determine if we're loading a CuDNNLSTM layer
561 # from the number of bias weights:
562 # CuDNNLSTM has (units * 8) weights; while LSTM has (units * 4)
563 # if there's no bias weight in the file, skip this conversion
564 units = weights[1].shape[0]
565 bias_shape = weights[2].shape
566 n_gates = 4
568 if bias_shape == (2 * units * n_gates,):
569 source = "CuDNNLSTM"
570 elif bias_shape == (units * n_gates,):
571 source = "LSTM"
572 else:
573 raise ValueError("Invalid bias shape: " + str(bias_shape))
575 def convert_lstm_weights(weights, from_cudnn=True):
576 """Converts the weights between CuDNNLSTM and LSTM.
578 Args:
579 weights: Original weights.
580 from_cudnn: Indicates whether original weights are from cuDNN
581 layer.
583 Returns:
584 Updated weights compatible with LSTM.
585 """
587 # Transpose (and reshape) input and recurrent kernels
588 kernels = transform_kernels(
589 weights[0], transpose_input(from_cudnn), n_gates
590 )
591 recurrent_kernels = transform_kernels(
592 weights[1], lambda k: k.T, n_gates
593 )
594 if from_cudnn:
595 # merge input and recurrent biases into a single set
596 biases = np.sum(np.split(weights[2], 2, axis=0), axis=0)
597 else:
598 # Split single set of biases evenly to two sets. The way of
599 # splitting doesn't matter as long as the two sets sum is kept.
600 biases = np.tile(0.5 * weights[2], 2)
601 return [kernels, recurrent_kernels, biases]
603 if source != target_class:
604 weights = convert_lstm_weights(
605 weights, from_cudnn=source == "CuDNNLSTM"
606 )
608 # convert the weights between CuDNNGRU and GRU(reset_after=True)
609 if target_class in ["GRU", "CuDNNGRU"] and len(weights) == 3:
610 # We can determine the source of the weights from the shape of the bias.
611 # If there is no bias we skip the conversion since
612 # CuDNNGRU always has biases.
614 units = weights[1].shape[0]
615 bias_shape = weights[2].shape
616 n_gates = 3
618 def convert_gru_weights(weights, from_cudnn=True):
619 """Converts the weights between CuDNNGRU and GRU.
621 Args:
622 weights: Original weights.
623 from_cudnn: Indicates whether original weights are from cuDNN
624 layer.
626 Returns:
627 Updated weights compatible with GRU.
628 """
630 kernels = transform_kernels(
631 weights[0], transpose_input(from_cudnn), n_gates
632 )
633 recurrent_kernels = transform_kernels(
634 weights[1], lambda k: k.T, n_gates
635 )
636 biases = np.array(weights[2]).reshape((2, -1) if from_cudnn else -1)
637 return [kernels, recurrent_kernels, biases]
639 if bias_shape == (2 * units * n_gates,):
640 source = "CuDNNGRU"
641 elif bias_shape == (2, units * n_gates):
642 source = "GRU(reset_after=True)"
643 elif bias_shape == (units * n_gates,):
644 source = "GRU(reset_after=False)"
645 else:
646 raise ValueError("Invalid bias shape: " + str(bias_shape))
648 if target_class == "CuDNNGRU":
649 target = "CuDNNGRU"
650 elif layer.reset_after:
651 target = "GRU(reset_after=True)"
652 else:
653 target = "GRU(reset_after=False)"
655 # only convert between different types
656 if source != target:
657 types = (source, target)
658 if "GRU(reset_after=False)" in types:
659 raise ValueError("%s is not compatible with %s" % types)
660 if source == "CuDNNGRU":
661 weights = convert_gru_weights(weights, from_cudnn=True)
662 elif source == "GRU(reset_after=True)":
663 weights = convert_gru_weights(weights, from_cudnn=False)
665 return weights
668def save_optimizer_weights_to_hdf5_group(hdf5_group, optimizer):
669 """Saves optimizer weights of a optimizer to a HDF5 group.
671 Args:
672 hdf5_group: HDF5 group.
673 optimizer: optimizer instance.
674 """
675 if isinstance(optimizer, optimizer_base.Optimizer):
676 symbolic_weights = optimizer.variables
677 else:
678 symbolic_weights = getattr(optimizer, "weights")
679 if symbolic_weights:
680 weights_group = hdf5_group.create_group("optimizer_weights")
681 weight_names = [str(w.name).encode("utf8") for w in symbolic_weights]
682 save_attributes_to_hdf5_group(
683 weights_group, "weight_names", weight_names
684 )
685 weight_values = backend.batch_get_value(symbolic_weights)
686 for name, val in zip(weight_names, weight_values):
687 param_dset = weights_group.create_dataset(
688 name, val.shape, dtype=val.dtype
689 )
690 if not val.shape:
691 # scalar
692 param_dset[()] = val
693 else:
694 param_dset[:] = val
697def load_optimizer_weights_from_hdf5_group(hdf5_group):
698 """Load optimizer weights from a HDF5 group.
700 Args:
701 hdf5_group: A pointer to a HDF5 group.
703 Returns:
704 data: List of optimizer weight names.
705 """
706 weights_group = hdf5_group["optimizer_weights"]
707 optimizer_weight_names = load_attributes_from_hdf5_group(
708 weights_group, "weight_names"
709 )
710 return [
711 weights_group[weight_name] for weight_name in optimizer_weight_names
712 ]
715def save_subset_weights_to_hdf5_group(f, weights):
716 """Save top-level weights of a model to a HDF5 group.
718 Args:
719 f: HDF5 group.
720 weights: List of weight variables.
721 """
722 weight_values = backend.batch_get_value(weights)
723 weight_names = [w.name.encode("utf8") for w in weights]
724 save_attributes_to_hdf5_group(f, "weight_names", weight_names)
725 for name, val in zip(weight_names, weight_values):
726 param_dset = f.create_dataset(name, val.shape, dtype=val.dtype)
727 if not val.shape:
728 # scalar
729 param_dset[()] = val
730 else:
731 param_dset[:] = val
734def save_weights_to_hdf5_group(f, model):
735 """Saves the weights of a list of layers to a HDF5 group.
737 Args:
738 f: HDF5 group.
739 model: Model instance.
740 """
741 from keras.src import __version__ as keras_version
743 save_attributes_to_hdf5_group(
744 f, "layer_names", [layer.name.encode("utf8") for layer in model.layers]
745 )
746 f.attrs["backend"] = backend.backend().encode("utf8")
747 f.attrs["keras_version"] = str(keras_version).encode("utf8")
749 # Sort model layers by layer name to ensure that group names are strictly
750 # growing to avoid prefix issues.
751 for layer in sorted(model.layers, key=lambda x: x.name):
752 g = f.create_group(layer.name)
753 weights = _legacy_weights(layer)
754 save_subset_weights_to_hdf5_group(g, weights)
755 weights = model._trainable_weights + model._non_trainable_weights
756 g = f.create_group("top_level_model_weights")
757 save_subset_weights_to_hdf5_group(g, weights)
760def load_subset_weights_from_hdf5_group(f):
761 """Load layer weights of a model from hdf5.
763 Args:
764 f: A pointer to a HDF5 group.
766 Returns:
767 List of NumPy arrays of the weight values.
769 Raises:
770 ValueError: in case of mismatch between provided model
771 and weights file.
772 """
773 weight_names = load_attributes_from_hdf5_group(f, "weight_names")
774 return [np.asarray(f[weight_name]) for weight_name in weight_names]
777def load_weights_from_hdf5_group(f, model):
778 """Implements topological (order-based) weight loading.
780 Args:
781 f: A pointer to a HDF5 group.
782 model: Model instance.
784 Raises:
785 ValueError: in case of mismatch between provided layers
786 and weights file.
787 """
788 if "keras_version" in f.attrs:
789 original_keras_version = f.attrs["keras_version"]
790 if hasattr(original_keras_version, "decode"):
791 original_keras_version = original_keras_version.decode("utf8")
792 else:
793 original_keras_version = "1"
794 if "backend" in f.attrs:
795 original_backend = f.attrs["backend"]
796 if hasattr(original_backend, "decode"):
797 original_backend = original_backend.decode("utf8")
798 else:
799 original_backend = None
801 filtered_layers = []
802 for layer in model.layers:
803 weights = _legacy_weights(layer)
804 if weights:
805 filtered_layers.append(layer)
807 layer_names = load_attributes_from_hdf5_group(f, "layer_names")
808 filtered_layer_names = []
809 for name in layer_names:
810 g = f[name]
811 weight_names = load_attributes_from_hdf5_group(g, "weight_names")
812 if weight_names:
813 filtered_layer_names.append(name)
814 layer_names = filtered_layer_names
815 if len(layer_names) != len(filtered_layers):
816 raise ValueError(
817 "Layer count mismatch when loading weights from file. "
818 f"Model expected {len(filtered_layers)} layers, found "
819 f"{len(layer_names)} saved layers."
820 )
822 # We batch weight value assignments in a single backend call
823 # which provides a speedup in TensorFlow.
824 weight_value_tuples = []
825 for k, name in enumerate(layer_names):
826 g = f[name]
827 layer = filtered_layers[k]
828 symbolic_weights = _legacy_weights(layer)
829 weight_values = load_subset_weights_from_hdf5_group(g)
830 weight_values = preprocess_weights_for_loading(
831 layer, weight_values, original_keras_version, original_backend
832 )
833 if len(weight_values) != len(symbolic_weights):
834 raise ValueError(
835 f"Weight count mismatch for layer #{k} (named {layer.name} in "
836 f"the current model, {name} in the save file). "
837 f"Layer expects {len(symbolic_weights)} weight(s). Received "
838 f"{len(weight_values)} saved weight(s)"
839 )
840 weight_value_tuples += zip(symbolic_weights, weight_values)
842 if "top_level_model_weights" in f:
843 symbolic_weights = (
844 model._trainable_weights + model._non_trainable_weights
845 )
846 weight_values = load_subset_weights_from_hdf5_group(
847 f["top_level_model_weights"]
848 )
849 if len(weight_values) != len(symbolic_weights):
850 raise ValueError(
851 "Weight count mismatch for top-level weights when loading "
852 "weights from file. "
853 f"Model expects {len(symbolic_weights)} top-level weight(s). "
854 f"Received {len(weight_values)} saved top-level weight(s)"
855 )
856 weight_value_tuples += zip(symbolic_weights, weight_values)
857 backend.batch_set_value(weight_value_tuples)
859 # Perform any layer defined finalization of the layer state.
860 for layer in model._flatten_layers():
861 layer.finalize_state()
864def load_weights_from_hdf5_group_by_name(f, model, skip_mismatch=False):
865 """Implements name-based weight loading (instead of topological loading).
867 Layers that have no matching name are skipped.
869 Args:
870 f: A pointer to a HDF5 group.
871 model: Model instance.
872 skip_mismatch: Boolean, whether to skip loading of layers
873 where there is a mismatch in the number of weights,
874 or a mismatch in the shape of the weights.
876 Raises:
877 ValueError: in case of mismatch between provided layers
878 and weights file and skip_match=False.
879 """
880 if "keras_version" in f.attrs:
881 original_keras_version = f.attrs["keras_version"]
882 if hasattr(original_keras_version, "decode"):
883 original_keras_version = original_keras_version.decode("utf8")
884 else:
885 original_keras_version = "1"
886 if "backend" in f.attrs:
887 original_backend = f.attrs["backend"]
888 if hasattr(original_backend, "decode"):
889 original_backend = original_backend.decode("utf8")
890 else:
891 original_backend = None
893 # New file format.
894 layer_names = load_attributes_from_hdf5_group(f, "layer_names")
896 # Reverse index of layer name to list of layers with name.
897 index = {}
898 for layer in model.layers:
899 if layer.name:
900 index.setdefault(layer.name, []).append(layer)
902 # We batch weight value assignments in a single backend call
903 # which provides a speedup in TensorFlow.
904 weight_value_tuples = []
905 for k, name in enumerate(layer_names):
906 g = f[name]
907 weight_values = load_subset_weights_from_hdf5_group(g)
908 for layer in index.get(name, []):
909 symbolic_weights = _legacy_weights(layer)
910 weight_values = preprocess_weights_for_loading(
911 layer, weight_values, original_keras_version, original_backend
912 )
913 if len(weight_values) != len(symbolic_weights):
914 if skip_mismatch:
915 logging.warning(
916 f"Skipping loading of weights for layer #{k} (named "
917 f"{layer.name}) due to mismatch in number of weights. "
918 f"Layer expects {len(symbolic_weights)} weight(s). "
919 f"Received {len(weight_values)} saved weight(s)"
920 )
921 continue
922 raise ValueError(
923 f"Weight count mismatch for layer #{k} "
924 f"(named {layer.name}). "
925 f"Layer expects {len(symbolic_weights)} weight(s). "
926 f"Received {len(weight_values)} saved weight(s)"
927 )
928 # Set values.
929 for i in range(len(weight_values)):
930 expected_shape = backend.int_shape(symbolic_weights[i])
931 received_shape = weight_values[i].shape
932 if expected_shape != received_shape:
933 if skip_mismatch:
934 logging.warning(
935 f"Skipping loading weights for layer #{k} (named "
936 f"{layer.name}) due to mismatch in shape for "
937 f"weight {symbolic_weights[i].name}. "
938 f"Weight expects shape {expected_shape}. "
939 "Received saved weight "
940 f"with shape {received_shape}"
941 )
942 continue
943 raise ValueError(
944 f"Shape mismatch in layer #{k} (named {layer.name}) "
945 f"for weight {symbolic_weights[i].name}. "
946 f"Weight expects shape {expected_shape}. "
947 "Received saved weight "
948 f"with shape {received_shape}"
949 )
950 else:
951 weight_value_tuples.append(
952 (symbolic_weights[i], weight_values[i])
953 )
955 if "top_level_model_weights" in f:
956 symbolic_weights = (
957 model._trainable_weights + model._non_trainable_weights
958 )
959 weight_values = load_subset_weights_from_hdf5_group(
960 f["top_level_model_weights"]
961 )
963 if len(weight_values) != len(symbolic_weights):
964 if skip_mismatch:
965 logging.warning(
966 "Skipping loading top-level weights for model due to "
967 "mismatch in number of weights. "
968 f"Model expects {len(symbolic_weights)} "
969 "top-level weight(s). "
970 f"Received {len(weight_values)} saved top-level weight(s)"
971 )
972 else:
973 raise ValueError(
974 "Weight count mismatch for top-level weights of model. "
975 f"Model expects {len(symbolic_weights)} "
976 "top-level weight(s). "
977 f"Received {len(weight_values)} saved top-level weight(s)"
978 )
979 else:
980 for i in range(len(weight_values)):
981 expected_shape = backend.int_shape(symbolic_weights[i])
982 received_shape = weight_values[i].shape
983 if expected_shape != received_shape:
984 if skip_mismatch:
985 logging.warning(
986 "Skipping loading top-level weight for model due "
987 "to mismatch in shape for "
988 f"weight {symbolic_weights[i].name}. "
989 f"Weight expects shape {expected_shape}. "
990 "Received saved weight "
991 f"with shape {received_shape}"
992 )
993 else:
994 raise ValueError(
995 "Shape mismatch in model for top-level weight "
996 f"{symbolic_weights[i].name}. "
997 f"Weight expects shape {expected_shape}. "
998 "Received saved weight "
999 f"with shape {received_shape}"
1000 )
1001 else:
1002 weight_value_tuples.append(
1003 (symbolic_weights[i], weight_values[i])
1004 )
1006 backend.batch_set_value(weight_value_tuples)
1008 # Perform any layer defined finalization of the layer state.
1009 for layer in model._flatten_layers():
1010 layer.finalize_state()
1013def save_attributes_to_hdf5_group(group, name, data):
1014 """Saves attributes (data) of the specified name into the HDF5 group.
1016 This method deals with an inherent problem of HDF5 file which is not
1017 able to store data larger than HDF5_OBJECT_HEADER_LIMIT bytes.
1019 Args:
1020 group: A pointer to a HDF5 group.
1021 name: A name of the attributes to save.
1022 data: Attributes data to store.
1024 Raises:
1025 RuntimeError: If any single attribute is too large to be saved.
1026 """
1027 # Check that no item in `data` is larger than `HDF5_OBJECT_HEADER_LIMIT`
1028 # because in that case even chunking the array would not make the saving
1029 # possible.
1030 bad_attributes = [x for x in data if len(x) > HDF5_OBJECT_HEADER_LIMIT]
1032 # Expecting this to never be true.
1033 if bad_attributes:
1034 raise RuntimeError(
1035 "The following attributes cannot be saved to HDF5 file because "
1036 f"they are larger than {HDF5_OBJECT_HEADER_LIMIT} "
1037 f"bytes: {bad_attributes}"
1038 )
1040 data_npy = np.asarray(data)
1042 num_chunks = 1
1043 chunked_data = np.array_split(data_npy, num_chunks)
1045 # This will never loop forever thanks to the test above.
1046 while any(x.nbytes > HDF5_OBJECT_HEADER_LIMIT for x in chunked_data):
1047 num_chunks += 1
1048 chunked_data = np.array_split(data_npy, num_chunks)
1050 if num_chunks > 1:
1051 for chunk_id, chunk_data in enumerate(chunked_data):
1052 group.attrs["%s%d" % (name, chunk_id)] = chunk_data
1053 else:
1054 group.attrs[name] = data
1057def load_attributes_from_hdf5_group(group, name):
1058 """Loads attributes of the specified name from the HDF5 group.
1060 This method deals with an inherent problem
1061 of HDF5 file which is not able to store
1062 data larger than HDF5_OBJECT_HEADER_LIMIT bytes.
1064 Args:
1065 group: A pointer to a HDF5 group.
1066 name: A name of the attributes to load.
1068 Returns:
1069 data: Attributes data.
1070 """
1071 if name in group.attrs:
1072 data = [
1073 n.decode("utf8") if hasattr(n, "decode") else n
1074 for n in group.attrs[name]
1075 ]
1076 else:
1077 data = []
1078 chunk_id = 0
1079 while "%s%d" % (name, chunk_id) in group.attrs:
1080 data.extend(
1081 [
1082 n.decode("utf8") if hasattr(n, "decode") else n
1083 for n in group.attrs["%s%d" % (name, chunk_id)]
1084 ]
1085 )
1086 chunk_id += 1
1087 return data
1090def _legacy_weights(layer):
1091 """DO NOT USE.
1093 For legacy reason, the layer.weights was in the order of
1094 [self.trainable_weights + self.non_trainable_weights], and this order was
1095 used for preserving the weights in h5 format. The new order of layer.weights
1096 are the same as layer.get_weights() which is more intuitive for user. To
1097 keep supporting the existing saved h5 file, this method should be used to
1098 save/load weights. In future version, we will delete this method and
1099 introduce a breaking change for h5 and stay with the new order for weights.
1101 Args:
1102 layer: a `tf.keras.Model` or `tf.keras.layers.Layer` instance.
1104 Returns:
1105 A list of variables with the order of trainable_weights, followed by
1106 non_trainable_weights.
1107 """
1108 weights = layer.trainable_weights + layer.non_trainable_weights
1109 if any(not isinstance(w, tf.Variable) for w in weights):
1110 raise NotImplementedError(
1111 "Save or restore weights that is not an instance of `tf.Variable` "
1112 "is not supported in h5, use `save_format='tf'` instead. Received "
1113 f"a model or layer {layer.__class__.__name__} "
1114 f"with weights {weights}"
1115 )
1116 return weights