Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/tensorflow/python/eager/backprop_util.py: 29%

1# Copyright 2015 The TensorFlow Authors. All Rights Reserved. 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================== 

15"""Shared utilities related to backprop.""" 

16 

17from tensorflow.core.config import flags 

18from tensorflow.core.framework import types_pb2 

19from tensorflow.python.framework import dtypes 

20from tensorflow.python.framework import indexed_slices 

21from tensorflow.python.framework import ops 

22from tensorflow.python.framework import tensor_util 

23from tensorflow.python.ops import array_ops 

24from tensorflow.python.ops import handle_data_util 

25from tensorflow.python.ops import math_ops 

26 

27 

28def _DTypeFromTensor(tensor): 

29 """Extract either `tensor.dtype` or the unanimous sub-type of a variant.""" 

30 dtype = tensor.dtype 

31 if dtype.base_dtype == dtypes.variant: 

32 # If we know statically that the data a variant points to is non-trainable 

33 # then the variant itself is non-trainable. 

34 if isinstance(tensor, ops.EagerTensor): 

35 handle_data = tensor._handle_data # pylint: disable=protected-access 

36 else: 

37 handle_data = handle_data_util.get_resource_handle_data(tensor) 

38 if (handle_data is not None 

39 and handle_data.is_set 

40 and handle_data.shape_and_type): 

41 first_type = handle_data.shape_and_type[0].dtype 

42 # Some variants have statically unknown dtypes; we can't make inferences 

43 # about trainability, so we conservatively assume they're trainable 

44 # (which may waste memory passing zeros around, but will be correct). 

45 if (first_type != types_pb2.DT_INVALID 

46 and all(shape_and_type.dtype == first_type 

47 for shape_and_type in handle_data.shape_and_type)): 

48 return first_type 

49 return dtype 

50 

51 

52def IsTrainable(tensor_or_dtype): 

53 """Determines whether a tensor or dtype supports infinitesimal changes.""" 

54 if tensor_util.is_tf_type(tensor_or_dtype): 

55 dtype = _DTypeFromTensor(tensor_or_dtype) 

56 else: 

57 dtype = tensor_or_dtype 

58 dtype = dtypes.as_dtype(dtype) 

59 trainable_dtypes = [dtypes.float16, dtypes.float32, dtypes.float64, 

60 dtypes.complex64, dtypes.complex128, dtypes.resource, 

61 dtypes.variant, dtypes.bfloat16] 

62 if flags.config().enable_quantized_dtypes_training.value(): 

63 trainable_dtypes.extend([dtypes.qint8, dtypes.qint16, dtypes.qint32, 

64 dtypes.quint8, dtypes.quint16]) 

65 return dtype.base_dtype in trainable_dtypes 

66 

67 

68def FlattenNestedIndexedSlices(grad): 

69 assert isinstance(grad, indexed_slices.IndexedSlices) 

70 if isinstance(grad.values, ops.Tensor): 

71 return grad 

72 else: 

73 assert isinstance(grad.values, indexed_slices.IndexedSlices) 

74 g = FlattenNestedIndexedSlices(grad.values) 

75 return indexed_slices.IndexedSlices( 

76 g.values, array_ops.gather(grad.indices, g.indices), g.dense_shape) 

77 

78 

79def AggregateIndexedSlicesGradients(grads): 

80 """Aggregates gradients containing `IndexedSlices`s.""" 

81 if len(grads) < 1: 

82 return None 

83 if len(grads) == 1: 

84 return grads[0] 

85 grads = [g for g in grads if g is not None] 

86 # If any gradient is a `Tensor`, sum them up and return a dense tensor 

87 # object. 

88 if any(isinstance(g, ops.Tensor) for g in grads): 

89 return math_ops.add_n(grads) 

90 

91 # The following `_as_indexed_slices_list` casts ids of IndexedSlices into 

92 # int64. It is to make sure the inputs of `concat` all have same the data 

93 # type. 

94 grads = math_ops._as_indexed_slices_list(grads) # pylint: disable=protected-access 

95 

96 grads = [FlattenNestedIndexedSlices(x) for x in grads] 

97 # Form IndexedSlices out of the concatenated values and indices. 

98 concat_grad = indexed_slices.IndexedSlices( 

99 array_ops.concat([x.values for x in grads], axis=0), 

100 array_ops.concat([x.indices for x in grads], axis=0), 

101 grads[0].dense_shape) 

102 

103 return concat_grad 

104