Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/tensorflow/python/keras/optimizer_v2/adagrad.py: 39%

1# Copyright 2018 The TensorFlow Authors. All Rights Reserved. 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================== 

15"""Adagrad optimizer implementation.""" 

16# pylint: disable=g-classes-have-attributes 

17 

18import numpy as np 

19 

20from tensorflow.python.framework import dtypes 

21from tensorflow.python.framework import tensor_conversion 

22from tensorflow.python.keras import backend_config 

23from tensorflow.python.keras.optimizer_v2 import optimizer_v2 

24from tensorflow.python.ops import array_ops 

25from tensorflow.python.ops import init_ops 

26from tensorflow.python.training import gen_training_ops 

27from tensorflow.python.util.tf_export import keras_export 

28 

29 

30@keras_export('keras.optimizers.Adagrad') 

31class Adagrad(optimizer_v2.OptimizerV2): 

32 r"""Optimizer that implements the Adagrad algorithm. 

33 

34 Adagrad is an optimizer with parameter-specific learning rates, 

35 which are adapted relative to how frequently a parameter gets 

36 updated during training. The more updates a parameter receives, 

37 the smaller the updates. 

38 

39 Args: 

40 learning_rate: Initial value for the learning rate: 

41 either a floating point value, 

42 or a `tf.keras.optimizers.schedules.LearningRateSchedule` instance. 

43 Defaults to 0.001. 

44 Note that `Adagrad` tends to benefit from higher initial learning rate 

45 values compared to other optimizers. 

46 To match the exact form in the original paper, use 1.0. 

47 initial_accumulator_value: Floating point value. 

48 Starting value for the accumulators (per-parameter momentum values). 

49 Must be non-negative. 

50 epsilon: Small floating point value used to maintain numerical stability. 

51 name: Optional name prefix for the operations created when applying 

52 gradients. Defaults to `"Adagrad"`. 

53 **kwargs: Keyword arguments. Allowed to be one of 

54 `"clipnorm"` or `"clipvalue"`. 

55 `"clipnorm"` (float) clips gradients by norm and represents 

56 the maximum L2 norm of each weight variable; 

57 `"clipvalue"` (float) clips gradient by value and represents the 

58 maximum absolute value of each weight variable. 

59 

60 Reference: 

61 - [Duchi et al., 2011]( 

62 http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf). 

63 """ 

64 

65 _HAS_AGGREGATE_GRAD = True 

66 

67 def __init__(self, 

68 learning_rate=0.001, 

69 initial_accumulator_value=0.1, 

70 epsilon=1e-7, 

71 name='Adagrad', 

72 **kwargs): 

73 if initial_accumulator_value < 0.0: 

74 raise ValueError('initial_accumulator_value must be non-negative: %s' % 

75 initial_accumulator_value) 

76 if epsilon is None: 

77 epsilon = backend_config.epsilon() 

78 super(Adagrad, self).__init__(name, **kwargs) 

79 self._set_hyper('learning_rate', kwargs.get('lr', learning_rate)) 

80 self._set_hyper('decay', self._initial_decay) 

81 self._initial_accumulator_value = initial_accumulator_value 

82 self.epsilon = epsilon or backend_config.epsilon() 

83 

84 def _create_slots(self, var_list): 

85 for var in var_list: 

86 dtype = var.dtype.base_dtype 

87 init = init_ops.constant_initializer( 

88 self._initial_accumulator_value, dtype=dtype) 

89 self.add_slot(var, 'accumulator', init) 

90 

91 def _prepare_local(self, var_device, var_dtype, apply_state): 

92 super(Adagrad, self)._prepare_local(var_device, var_dtype, apply_state) 

93 apply_state[(var_device, var_dtype)].update( 

94 dict( 

95 epsilon=tensor_conversion.convert_to_tensor_v2_with_dispatch( 

96 self.epsilon, var_dtype 

97 ), 

98 neg_lr_t=-apply_state[(var_device, var_dtype)]['lr_t'], 

99 zero=array_ops.zeros((), dtype=dtypes.int64), 

100 ) 

101 ) 

102 

103 def set_weights(self, weights): 

104 params = self.weights 

105 # Override set_weights for backward compatibility of Keras V1 optimizer 

106 # since it does not include iteration at head of the weight list. Set 

107 # iteration to 0. 

108 if len(params) == len(weights) + 1: 

109 weights = [np.array(0)] + weights 

110 super(Adagrad, self).set_weights(weights) 

111 

112 @classmethod 

113 def from_config(cls, config, custom_objects=None): 

114 """Creates an optimizer from its config. 

115 

116 This method is the reverse of `get_config`, 

117 capable of instantiating the same optimizer from the config 

118 dictionary. 

119 

120 Args: 

121 config: A Python dictionary, typically the output of get_config. 

122 custom_objects: A Python dictionary mapping names to additional Python 

123 objects used to create this optimizer, such as a function used for a 

124 hyperparameter. 

125 

126 Returns: 

127 An optimizer instance. 

128 """ 

129 if 'initial_accumulator_value' not in config: 

130 config['initial_accumulator_value'] = 0.1 

131 if 'lr' in config: 

132 config['learning_rate'] = config.pop('lr') 

133 return cls(**config) 

134 

135 def _resource_apply_dense(self, grad, var, apply_state=None): 

136 var_device, var_dtype = var.device, var.dtype.base_dtype 

137 coefficients = ((apply_state or {}).get((var_device, var_dtype)) 

138 or self._fallback_apply_state(var_device, var_dtype)) 

139 

140 acc = self.get_slot(var, 'accumulator') 

141 return gen_training_ops.ResourceApplyAdagradV2( 

142 var=var.handle, 

143 accum=acc.handle, 

144 lr=coefficients['lr_t'], 

145 epsilon=coefficients['epsilon'], 

146 grad=grad, 

147 use_locking=self._use_locking) 

148 

149 def _resource_apply_sparse(self, grad, var, indices, apply_state=None): 

150 var_device, var_dtype = var.device, var.dtype.base_dtype 

151 coefficients = ((apply_state or {}).get((var_device, var_dtype)) 

152 or self._fallback_apply_state(var_device, var_dtype)) 

153 

154 acc = self.get_slot(var, 'accumulator') 

155 return gen_training_ops.ResourceSparseApplyAdagradV2( 

156 var=var.handle, 

157 accum=acc.handle, 

158 lr=coefficients['lr_t'], 

159 epsilon=coefficients['epsilon'], 

160 grad=grad, 

161 indices=indices, 

162 use_locking=self._use_locking) 

163 

164 def get_config(self): 

165 config = super(Adagrad, self).get_config() 

166 config.update({ 

167 'learning_rate': self._serialize_hyperparameter('learning_rate'), 

168 'decay': self._initial_decay, 

169 'initial_accumulator_value': self._initial_accumulator_value, 

170 'epsilon': self.epsilon, 

171 }) 

172 return config