Grcov report - yaml

1

#include <limits>

2

#include <numeric>

3

4

#include "envoy/annotations/deprecation.pb.h"

5

#include "envoy/protobuf/message_validator.h"

6

#include "envoy/type/v3/percent.pb.h"

7

8

#include "source/common/common/assert.h"

9

#include "source/common/common/documentation_url.h"

10

#include "source/common/common/fmt.h"

11

#include "source/common/protobuf/message_validator_impl.h"

12

#include "source/common/protobuf/protobuf.h"

13

#include "source/common/protobuf/utility.h"

14

#include "source/common/protobuf/visitor.h"

15

#include "source/common/runtime/runtime_features.h"

16

17

#include "absl/strings/match.h"

18

#include "udpa/annotations/sensitive.pb.h"

19

#include "udpa/annotations/status.pb.h"

20

#include "utf8_validity.h"

21

#include "validate/validate.h"

22

#include "xds/annotations/v3/status.pb.h"

23

#include "yaml-cpp/yaml.h"

24

25

using namespace std::chrono_literals;

26

27

namespace Envoy {

28

namespace {

29

30

11462

void blockFormat(YAML::Node node) {

31

11462

  node.SetStyle(YAML::EmitterStyle::Block);

32

33

11462

  if (node.Type() == YAML::NodeType::Sequence) {

34

1648

    for (const auto& it : node) {

35

1648

      blockFormat(it);

36

1648

37

1275

38

11462

  if (node.Type() == YAML::NodeType::Map) {

39

9556

    for (const auto& it : node) {

40

9500

      blockFormat(it.second);

41

9500

42

5343

43

11462

44

45

1186241

Protobuf::Value parseYamlNode(const YAML::Node& node) {

46

1186241

  Protobuf::Value value;

47

1186241

  switch (node.Type()) {

48

446

  case YAML::NodeType::Null:

49

446

    value.set_null_value(Protobuf::NULL_VALUE);

50

446

    break;

51

505275

  case YAML::NodeType::Scalar: {

52

505275

    if (node.Tag() == "!") {

53

122278

      value.set_string_value(node.as<std::string>());

54

122278

      break;

55

122278

56

382997

    bool bool_value;

57

382997

    if (YAML::convert<bool>::decode(node, bool_value)) {

58

11011

      value.set_bool_value(bool_value);

59

11011

      break;

60

11011

61

371986

    int64_t int_value;

62

371986

    if (YAML::convert<int64_t>::decode(node, int_value)) {

63

57838

      if (std::numeric_limits<int32_t>::min() <= int_value &&

64

57838

          std::numeric_limits<int32_t>::max() >= int_value) {

65

        // We could convert all integer values to string but it will break some stuff relying on

66

        // Protobuf::Struct itself, only convert small numbers into number_value here.

67

57803

        value.set_number_value(int_value);

68

57818

      } else {

69

        // Proto3 JSON mapping allows use string for integer, this still has to be converted from

70

        // int_value to support hexadecimal and octal literals.

71

35

        value.set_string_value(std::to_string(int_value));

72

35

73

57838

      break;

74

57838

75

    // Fall back on string, including float/double case. When protobuf parse the JSON into a message

76

    // it will convert based on the type in the message definition.

77

314148

    value.set_string_value(node.as<std::string>());

78

314148

    break;

79

371986

80

91305

  case YAML::NodeType::Sequence: {

81

91305

    auto& list_values = *value.mutable_list_value()->mutable_values();

82

106349

    for (const auto& it : node) {

83

106202

      *list_values.Add() = parseYamlNode(it);

84

106202

85

91305

    break;

86

371986

87

589215

  case YAML::NodeType::Map: {

88

589215

    auto& struct_fields = *value.mutable_struct_value()->mutable_fields();

89

1044909

    for (const auto& it : node) {

90

1044718

      if (it.first.Tag() != "!ignore") {

91

1044717

        struct_fields[it.first.as<std::string>()] = parseYamlNode(it.second);

92

1044717

93

1044718

94

589215

    break;

95

371986

96

  case YAML::NodeType::Undefined:

97

    throw EnvoyException("Undefined YAML value");

98

1186241

99

1186231

  return value;

100

1186241

101

102

void jsonConvertInternal(const Protobuf::Message& source,

103

                         ProtobufMessage::ValidationVisitor& validation_visitor,

104

31275

                         Protobuf::Message& dest) {

105

31275

  Protobuf::util::JsonPrintOptions json_options;

106

31275

  json_options.preserve_proto_field_names = true;

107

31275

  std::string json;

108

31275

  const auto status = Protobuf::util::MessageToJsonString(source, &json, json_options);

109

31275

  if (!status.ok()) {

110

1

    throw EnvoyException(fmt::format("Unable to convert protobuf message to JSON string: {} {}",

111

1

                                     status.ToString(), source.DebugString()));

112

1

113

31274

  MessageUtil::loadFromJson(json, dest, validation_visitor);

114

31274

115

116

} // namespace

117

118

void MessageUtil::loadFromJson(absl::string_view json, Protobuf::Message& message,

119

42521

                               ProtobufMessage::ValidationVisitor& validation_visitor) {

120

42521

  bool has_unknown_field;

121

42521

  auto load_status = loadFromJsonNoThrow(json, message, has_unknown_field);

122

42521

  if (load_status.ok()) {

123

42396

    return;

124

42396

125

  if (has_unknown_field) {

126

    // If the parsing failure is caused by the unknown fields.

127

66

    THROW_IF_NOT_OK(validation_visitor.onUnknownField(

128

66

        fmt::format("type {} reason {}", message.GetTypeName(), load_status.ToString())));

129

114

  } else {

130

    // If the error has nothing to do with unknown field.

131

59

    throw EnvoyException(

132

59

        fmt::format("Unable to parse JSON as proto ({}): {}", load_status.ToString(), json));

133

59

134

125

135

136

absl::Status MessageUtil::loadFromJsonNoThrow(absl::string_view json, Protobuf::Message& message,

137

42670

                                              bool& has_unknown_fileld) {

138

42670

  has_unknown_fileld = false;

139

42670

  Protobuf::util::JsonParseOptions options;

140

42670

  options.case_insensitive_enum_parsing = true;

141

  // Let's first try and get a clean parse when checking for unknown fields;

142

  // this should be the common case.

143

42670

  options.ignore_unknown_fields = false;

144

  // Clear existing values (if any) from the destination message before serialization.

145

42670

  message.Clear();

146

42670

  const auto strict_status = Protobuf::util::JsonStringToMessage(json, &message, options);

147

42670

  if (strict_status.ok()) {

148

    // Success, no need to do any extra work.

149

42539

    return strict_status;

150

42539

151

  // If we fail, we see if we get a clean parse when allowing unknown fields.

152

  // This is essentially a workaround

153

  // for https://github.com/protocolbuffers/protobuf/issues/5967.

154

  // TODO(htuch): clean this up when protobuf supports JSON/YAML unknown field

155

  // detection directly.

156

131

  options.ignore_unknown_fields = true;

157

131

  const auto relaxed_status = Protobuf::util::JsonStringToMessage(json, &message, options);

158

  // If we still fail with relaxed unknown field checking, the error has nothing

159

  // to do with unknown fields.

160

131

  if (relaxed_status.ok()) {

161

67

    has_unknown_fileld = true;

162

67

    return strict_status;

163

67

164

64

  return relaxed_status;

165

131

166

167

23

absl::Status MessageUtil::loadFromJsonNoThrow(absl::string_view json, Protobuf::Struct& message) {

168

23

  message.Clear();

169

23

  return Protobuf::util::JsonStringToMessage(json, &message);

170

23

171

172

413

void MessageUtil::loadFromJson(absl::string_view json, Protobuf::Struct& message) {

173

  // No need to validate if converting to a Struct, since there are no unknown

174

  // fields possible.

175

413

  loadFromJson(json, message, ProtobufMessage::getNullValidationVisitor());

176

413

177

178

void MessageUtil::loadFromYaml(const std::string& yaml, Protobuf::Message& message,

179

31111

                               ProtobufMessage::ValidationVisitor& validation_visitor) {

180

31111

  Protobuf::Value value = ValueUtil::loadFromYaml(yaml);

181

31111

  if (value.kind_case() == Protobuf::Value::kStructValue ||

182

31111

      value.kind_case() == Protobuf::Value::kListValue) {

183

31108

    jsonConvertInternal(value, validation_visitor, message);

184

31108

    return;

185

31108

186

3

  throw EnvoyException("Unable to convert YAML as JSON: " + yaml);

187

31111

188

189

std::string MessageUtil::getYamlStringFromMessage(const Protobuf::Message& message,

190

                                                  const bool block_print,

191

316

                                                  const bool always_print_primitive_fields) {

192

193

316

  auto json_or_error = getJsonStringFromMessage(message, false, always_print_primitive_fields);

194

316

  if (!json_or_error.ok()) {

195

1

    throw EnvoyException(json_or_error.status().ToString());

196

1

197

315

  YAML::Node node;

198

315

  TRY_ASSERT_MAIN_THREAD { node = YAML::Load(json_or_error.value()); }

199

315

  END_TRY

200

315

  catch (YAML::ParserException& e) {

201

    throw EnvoyException(e.what());

202

203

315

  catch (YAML::BadConversion& e) {

204

    throw EnvoyException(e.what());

205

206

315

  catch (std::exception& e) {

207

    // Catch unknown YAML parsing exceptions.

208

    throw EnvoyException(fmt::format("Unexpected YAML exception: {}", +e.what()));

209

210

315

  if (block_print) {

211

314

    blockFormat(node);

212

314

213

315

  YAML::Emitter out;

214

315

  out << node;

215

315

  return out.c_str();

216

315

217

218

absl::StatusOr<std::string>

219

MessageUtil::getJsonStringFromMessage(const Protobuf::Message& message, const bool pretty_print,

220

1123949

                                      const bool always_print_primitive_fields) {

221

1123949

  Protobuf::util::JsonPrintOptions json_options;

222

  // By default, proto field names are converted to camelCase when the message is converted to JSON.

223

  // Setting this option makes debugging easier because it keeps field names consistent in JSON

224

  // printouts.

225

1123949

  json_options.preserve_proto_field_names = true;

226

1123949

  if (pretty_print) {

227

270

    json_options.add_whitespace = true;

228

270

229

  // Primitive types such as int32s and enums will not be serialized if they have the default value.

230

  // This flag disables that behavior.

231

1123949

  if (always_print_primitive_fields) {

232

1112245

    json_options.always_print_fields_with_no_presence = true;

233

1112245

234

1123949

  std::string json;

235

1123949

  if (auto status = Protobuf::util::MessageToJsonString(message, &json, json_options);

236

1123949

      !status.ok()) {

237

3

    return status;

238

3

239

1123946

  return json;

240

1123949

241

242

std::string MessageUtil::getJsonStringFromMessageOrError(const Protobuf::Message& message,

243

                                                         bool pretty_print,

244

1123529

                                                         bool always_print_primitive_fields) {

245

1123529

  auto json_or_error =

246

1123529

      getJsonStringFromMessage(message, pretty_print, always_print_primitive_fields);

247

1123529

  return json_or_error.ok() ? std::move(json_or_error).value()

248

1123529

                            : fmt::format("Failed to convert protobuf message to JSON string: {}",

249

1

                                          json_or_error.status().ToString());

250

1123529

251

252

22

void MessageUtil::jsonConvert(const Protobuf::Message& source, Protobuf::Message& dest) {

253

22

  jsonConvertInternal(source, ProtobufMessage::getNullValidationVisitor(), dest);

254

22

255

256

47

void MessageUtil::jsonConvert(const Protobuf::Message& source, Protobuf::Struct& dest) {

257

  // Any proto3 message can be transformed to Struct, so there is no need to check for unknown

258

  // fields. There is one catch; Duration/Timestamp etc. which have non-object canonical JSON

259

  // representations don't work.

260

47

  jsonConvertInternal(source, ProtobufMessage::getNullValidationVisitor(), dest);

261

47

262

263

void MessageUtil::jsonConvert(const Protobuf::Struct& source,

264

                              ProtobufMessage::ValidationVisitor& validation_visitor,

265

98

                              Protobuf::Message& dest) {

266

98

  jsonConvertInternal(source, validation_visitor, dest);

267

98

268

269

35

bool MessageUtil::jsonConvertValue(const Protobuf::Message& source, Protobuf::Value& dest) {

270

35

  Protobuf::util::JsonPrintOptions json_options;

271

35

  json_options.preserve_proto_field_names = true;

272

35

  std::string json;

273

35

  auto status = Protobuf::util::MessageToJsonString(source, &json, json_options);

274

35

  if (!status.ok()) {

275

2

    return false;

276

2

277

33

  bool has_unknow_field;

278

33

  status = MessageUtil::loadFromJsonNoThrow(json, dest, has_unknow_field);

279

33

  if (status.ok() || has_unknow_field) {

280

33

    return true;

281

33

282

  return false;

283

33

284

285

35324

Protobuf::Value ValueUtil::loadFromYaml(const std::string& yaml) {

286

35324

  TRY_ASSERT_MAIN_THREAD { return parseYamlNode(YAML::Load(yaml)); }

287

35324

  END_TRY

288

35324

  catch (YAML::ParserException& e) {

289

2

    throw EnvoyException(e.what());

290

2

291

35324

  catch (YAML::BadConversion& e) {

292

2

    throw EnvoyException(e.what());

293

2

294

35324

  catch (std::exception& e) {

295

    // There is a potentially wide space of exceptions thrown by the YAML parser,

296

    // and enumerating them all may be difficult. Envoy doesn't work well with

297

    // unhandled exceptions, so we capture them and record the exception name in

298

    // the Envoy Exception text.

299

    throw EnvoyException(fmt::format("Unexpected YAML exception: {}", +e.what()));

300

301

35324

302

303

namespace {

304

305

// This is a modified copy of the UTF8CoerceToStructurallyValid from the protobuf code.

306

// A copy was needed after if was removed from the protobuf.

307

3797

std::string utf8CoerceToStructurallyValid(absl::string_view str, const char replace_char) {

308

3797

  std::string result(str);

309

3797

  auto replace_pos = result.begin();

310

3835

  while (!str.empty()) {

311

3645

    size_t n_valid_bytes = utf8_range::SpanStructurallyValid(str);

312

3645

    if (n_valid_bytes == str.size()) {

313

3607

      break;

314

3607

315

38

    replace_pos += n_valid_bytes;

316

38

    *replace_pos++ = replace_char; // replace one bad byte

317

38

    str.remove_prefix(n_valid_bytes + 1);

318

38

319

3797

  return result;

320

3797

321

322

} // namespace

323

324

3797

std::string MessageUtil::sanitizeUtf8String(absl::string_view input) {

325

  // The choice of '!' is somewhat arbitrary, but we wanted to avoid any character that has

326

  // special semantic meaning in URLs or similar.

327

3797

  return utf8CoerceToStructurallyValid(input, '!');

328

3797

329

330

} // namespace Envoy