Grcov report - deterministic

1

#if defined(ENVOY_ENABLE_FULL_PROTOS)

2

#include "source/common/protobuf/deterministic_hash.h"

3

4

#include "source/common/common/assert.h"

5

#include "source/common/common/hash.h"

6

7

namespace Envoy {

8

namespace DeterministicProtoHash {

9

namespace {

10

11

// Get a scalar field from protobuf reflection field definition. The return

12

// type must be specified by the caller. Every implementation is a specialization

13

// because the reflection interface did separate named functions instead of a

14

// template.

15

template <typename T>

16

T reflectionGet(const Protobuf::Reflection& reflection, const Protobuf::Message& message,

17

                const Protobuf::FieldDescriptor& field);

18

19

template <>

20

uint32_t reflectionGet(const Protobuf::Reflection& reflection, const Protobuf::Message& message,

21

58197

                       const Protobuf::FieldDescriptor& field) {

22

58197

  return reflection.GetUInt32(message, &field);

23

58197

24

25

template <>

26

int32_t reflectionGet(const Protobuf::Reflection& reflection, const Protobuf::Message& message,

27

9929

                      const Protobuf::FieldDescriptor& field) {

28

9929

  return reflection.GetInt32(message, &field);

29

9929

30

31

template <>

32

uint64_t reflectionGet(const Protobuf::Reflection& reflection, const Protobuf::Message& message,

33

347

                       const Protobuf::FieldDescriptor& field) {

34

347

  return reflection.GetUInt64(message, &field);

35

347

36

37

template <>

38

int64_t reflectionGet(const Protobuf::Reflection& reflection, const Protobuf::Message& message,

39

24851

                      const Protobuf::FieldDescriptor& field) {

40

24851

  return reflection.GetInt64(message, &field);

41

24851

42

43

template <>

44

float reflectionGet(const Protobuf::Reflection& reflection, const Protobuf::Message& message,

45

4

                    const Protobuf::FieldDescriptor& field) {

46

4

  return reflection.GetFloat(message, &field);

47

4

48

49

template <>

50

double reflectionGet(const Protobuf::Reflection& reflection, const Protobuf::Message& message,

51

156

                     const Protobuf::FieldDescriptor& field) {

52

156

  return reflection.GetDouble(message, &field);

53

156

54

55

template <>

56

bool reflectionGet(const Protobuf::Reflection& reflection, const Protobuf::Message& message,

57

7994

                   const Protobuf::FieldDescriptor& field) {

58

7994

  return reflection.GetBool(message, &field);

59

7994

60

61

// Takes a field of scalar type, and hashes it. In case the field is a repeated field,

62

// the function hashes each of its elements.

63

template <typename T, std::enable_if_t<std::is_scalar_v<T>, bool> = true>

64

uint64_t hashScalarField(const Protobuf::Reflection& reflection, const Protobuf::Message& message,

65

101635

                         const Protobuf::FieldDescriptor& field, uint64_t seed) {

66

101635

  if (field.is_repeated()) {

67

187

    for (const T& scalar : reflection.GetRepeatedFieldRef<T>(message, &field)) {

68

187

      seed = HashUtil::xxHash64Value(scalar, seed);

69

187

70

101478

  } else {

71

101478

    seed = HashUtil::xxHash64Value(reflectionGet<T>(reflection, message, field), seed);

72

101478

73

101635

  return seed;

74

101635

75

76

uint64_t reflectionHashMessage(const Protobuf::Message& message, uint64_t seed = 0);

77

uint64_t reflectionHashField(const Protobuf::Message& message,

78

                             const Protobuf::FieldDescriptor& field, uint64_t seed);

79

80

// To make a map serialize deterministically we need to ignore the order of

81

// the map fields. To do that, we simply combine the hashes of each entry

82

// using an unordered operator (addition), and then apply that combined hash to

83

// the seed.

84

uint64_t reflectionHashMapField(const Protobuf::Message& message,

85

20091

                                const Protobuf::FieldDescriptor& field, uint64_t seed) {

86

20091

  const Protobuf::Reflection& reflection = *message.GetReflection();

87

20091

  ASSERT(field.is_map());

88

20091

  const auto& entries = reflection.GetRepeatedFieldRef<Protobuf::Message>(message, &field);

89

20091

  ASSERT(!entries.empty());

90

20091

  const Protobuf::Descriptor& map_descriptor = *entries.begin()->GetDescriptor();

91

20091

  const Protobuf::FieldDescriptor& key_field = *map_descriptor.map_key();

92

20091

  const Protobuf::FieldDescriptor& value_field = *map_descriptor.map_value();

93

20091

  uint64_t combined_hash = 0;

94

23247

  for (const Protobuf::Message& entry : entries) {

95

23247

    uint64_t entry_hash = reflectionHashField(entry, key_field, 0);

96

23247

    entry_hash = reflectionHashField(entry, value_field, entry_hash);

97

23247

    combined_hash += entry_hash;

98

23247

99

20091

  return HashUtil::xxHash64Value(combined_hash, seed);

100

20091

101

102

uint64_t reflectionHashField(const Protobuf::Message& message,

103

996694

                             const Protobuf::FieldDescriptor& field, uint64_t seed) {

104

996694

  using Protobuf::FieldDescriptor;

105

996694

  const Protobuf::Reflection& reflection = *message.GetReflection();

106

996694

  seed = HashUtil::xxHash64Value(field.number(), seed);

107

996694

  switch (field.cpp_type()) {

108

10017

  case FieldDescriptor::CPPTYPE_INT32:

109

10017

    seed = hashScalarField<int32_t>(reflection, message, field, seed);

110

10017

    break;

111

58244

  case FieldDescriptor::CPPTYPE_UINT32:

112

58244

    seed = hashScalarField<uint32_t>(reflection, message, field, seed);

113

58244

    break;

114

24855

  case FieldDescriptor::CPPTYPE_INT64:

115

24855

    seed = hashScalarField<int64_t>(reflection, message, field, seed);

116

24855

    break;

117

351

  case FieldDescriptor::CPPTYPE_UINT64:

118

351

    seed = hashScalarField<uint64_t>(reflection, message, field, seed);

119

351

    break;

120

160

  case FieldDescriptor::CPPTYPE_DOUBLE:

121

160

    seed = hashScalarField<double>(reflection, message, field, seed);

122

160

    break;

123

8

  case FieldDescriptor::CPPTYPE_FLOAT:

124

8

    seed = hashScalarField<float>(reflection, message, field, seed);

125

8

    break;

126

8000

  case FieldDescriptor::CPPTYPE_BOOL:

127

8000

    seed = hashScalarField<bool>(reflection, message, field, seed);

128

8000

    break;

129

25473

  case FieldDescriptor::CPPTYPE_ENUM:

130

25473

    if (field.is_repeated()) {

131

7

      const int c = reflection.FieldSize(message, &field);

132

18

      for (int i = 0; i < c; i++) {

133

11

        seed = HashUtil::xxHash64Value(reflection.GetRepeatedEnumValue(message, &field, i), seed);

134

11

135

25466

    } else {

136

25466

      seed = HashUtil::xxHash64Value(reflection.GetEnumValue(message, &field), seed);

137

25466

138

25473

    break;

139

312538

  case FieldDescriptor::CPPTYPE_STRING:

140

312538

    if (field.is_repeated()) {

141

15849

      for (const std::string& str : reflection.GetRepeatedFieldRef<std::string>(message, &field)) {

142

15849

        seed = HashUtil::xxHash64(str, seed);

143

15849

144

297676

    } else {

145

      // Scratch may be used by GetStringReference if the field is not already a std::string.

146

297676

      std::string scratch;

147

297676

      seed = HashUtil::xxHash64(reflection.GetStringReference(message, &field, &scratch), seed);

148

297676

149

312538

    break;

150

557048

  case FieldDescriptor::CPPTYPE_MESSAGE:

151

557048

    if (field.is_map()) {

152

20091

      seed = reflectionHashMapField(message, field, seed);

153

536986

    } else if (field.is_repeated()) {

154

126215

      for (const Protobuf::Message& submsg :

155

149533

           reflection.GetRepeatedFieldRef<Protobuf::Message>(message, &field)) {

156

149533

        seed = reflectionHashMessage(submsg, seed);

157

149533

158

410742

    } else {

159

410742

      seed = reflectionHashMessage(reflection.GetMessage(message, &field), seed);

160

410742

161

557048

    break;

162

996694

163

996694

  return seed;

164

996694

165

166

// Converts from type urls OR descriptor full names to descriptor full names.

167

// Type urls are as used in envoy yaml config, e.g.

168

// "type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.v3.UdpProxyConfig"

169

// becomes

170

// "envoy.extensions.filters.udp.udp_proxy.v3.UdpProxyConfig"

171

64259

absl::string_view typeUrlToDescriptorFullName(absl::string_view url) {

172

64259

  const size_t pos = url.rfind('/');

173

64259

  if (pos != absl::string_view::npos) {

174

64222

    return url.substr(pos + 1);

175

64222

176

37

  return url;

177

64259

178

179

64259

std::unique_ptr<Protobuf::Message> unpackAnyForReflection(const Protobuf::Any& any) {

180

64259

  const Protobuf::Descriptor* descriptor =

181

64259

      Protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName(

182

64259

          typeUrlToDescriptorFullName(any.type_url()));

183

  // If the type name refers to an unknown type, we treat it the same as other

184

  // unknown fields - not including its contents in the hash.

185

64259

  if (descriptor == nullptr) {

186

38

    return nullptr;

187

38

188

64221

  const Protobuf::Message* prototype =

189

64221

      Protobuf::MessageFactory::generated_factory()->GetPrototype(descriptor);

190

64221

  ASSERT(prototype != nullptr, "should be impossible since the descriptor is known");

191

64221

  std::unique_ptr<Protobuf::Message> msg(prototype->New());

192

64221

  if (!any.UnpackTo(msg.get())) {

193

2

    return nullptr;

194

2

195

64219

  return msg;

196

64221

197

198

// This is intentionally ignoring unknown fields.

199

665027

uint64_t reflectionHashMessage(const Protobuf::Message& message, uint64_t seed) {

200

665027

  using Protobuf::FieldDescriptor;

201

665027

  const Protobuf::Reflection* reflection = message.GetReflection();

202

665027

  const Protobuf::Descriptor* descriptor = message.GetDescriptor();

203

665027

  seed = HashUtil::xxHash64(descriptor->full_name(), seed);

204

665027

  if (descriptor->well_known_type() == Protobuf::Descriptor::WELLKNOWNTYPE_ANY) {

205

64259

    const Protobuf::Any* any = Protobuf::DynamicCastMessage<Protobuf::Any>(&message);

206

64259

    ASSERT(any != nullptr, "casting to any should always work for WELLKNOWNTYPE_ANY");

207

64259

    std::unique_ptr<Protobuf::Message> submsg = unpackAnyForReflection(*any);

208

64259

    if (submsg == nullptr) {

209

      // If we wanted to handle unknown types in Any, this is where we'd have to do it.

210

      // Since we don't know the type to introspect it, we hash just its type name.

211

40

      return HashUtil::xxHash64(any->type_url(), seed);

212

40

213

64219

    return reflectionHashMessage(*submsg, seed);

214

64259

215

600768

  std::vector<const FieldDescriptor*> fields;

216

  // ListFields returned the fields ordered by field number.

217

600768

  reflection->ListFields(message, &fields);

218

  // If we wanted to handle unknown fields, we'd need to also GetUnknownFields here.

219

950947

  for (const FieldDescriptor* field : fields) {

220

950200

    seed = reflectionHashField(message, *field, seed);

221

950200

222

  // Hash one extra character to signify end of message, so that

223

  // msg{} field2=2

224

  // hashes differently from

225

  // msg{field2=2}

226

600768

  return HashUtil::xxHash64("\x17", seed);

227

665027

228

} // namespace

229

230

40533

uint64_t hash(const Protobuf::Message& message) { return reflectionHashMessage(message, 0); }

231

232

} // namespace DeterministicProtoHash

233

} // namespace Envoy

234

#endif