Grcov report - utility.cc

1

#include "source/common/protobuf/utility.h"

2

3

#include <limits>

4

#include <numeric>

5

6

#include "envoy/annotations/deprecation.pb.h"

7

#include "envoy/protobuf/message_validator.h"

8

#include "envoy/type/v3/percent.pb.h"

9

10

#include "source/common/common/assert.h"

11

#include "source/common/common/documentation_url.h"

12

#include "source/common/common/fmt.h"

13

#include "source/common/protobuf/deterministic_hash.h"

14

#include "source/common/protobuf/message_validator_impl.h"

15

#include "source/common/protobuf/protobuf.h"

16

#include "source/common/protobuf/visitor.h"

17

#include "source/common/runtime/runtime_features.h"

18

19

#include "absl/strings/match.h"

20

#include "udpa/annotations/sensitive.pb.h"

21

#include "udpa/annotations/status.pb.h"

22

#include "validate/validate.h"

23

#include "xds/annotations/v3/status.pb.h"

24

25

using namespace std::chrono_literals;

26

27

namespace Envoy {

28

namespace {

29

30

// Validates that the max value of nanoseconds and seconds doesn't cause an

31

// overflow in the protobuf time-util computations.

32

94463

absl::Status validateDurationNoThrow(const Protobuf::Duration& duration) {

33

  // Apply a strict max boundary to the `seconds` value to avoid overflow when

34

  // both seconds and nanoseconds are at their highest values.

35

  // Note that protobuf internally converts to the input's seconds and

36

  // nanoseconds to nanoseconds (with a max nanoseconds value of 999999999).

37

  // The kMaxSecondsValue = 9223372035, which is about 292 years.

38

94463

  constexpr int64_t kMaxSecondsValue =

39

94463

      (std::numeric_limits<int64_t>::max() - 999999999) / (1000 * 1000 * 1000);

40

41

94463

  if (duration.seconds() < 0 || duration.nanos() < 0) {

42

13

    return absl::OutOfRangeError(

43

13

        fmt::format("Invalid duration: Expected positive duration: {}", duration.DebugString()));

44

13

45

94450

  if (!Protobuf::util::TimeUtil::IsDurationValid(duration)) {

46

6

    return absl::OutOfRangeError(

47

6

        fmt::format("Invalid duration: Duration out-of-range according to Protobuf: {}",

48

6

                    duration.DebugString()));

49

6

50

94444

  if (duration.nanos() > 999999999 || duration.seconds() > kMaxSecondsValue) {

51

6

    return absl::OutOfRangeError(

52

6

        fmt::format("Invalid duration: Duration out-of-range: {}", duration.DebugString()));

53

6

54

94438

  return absl::OkStatus();

55

94444

56

57

50866

void validateDuration(const Protobuf::Duration& duration) {

58

50866

  const absl::Status result = validateDurationNoThrow(duration);

59

50866

  if (!result.ok()) {

60

12

    throwEnvoyExceptionOrPanic(std::string(result.message()));

61

12

62

50866

63

64

13289391

absl::string_view filenameFromPath(absl::string_view full_path) {

65

13289391

  size_t index = full_path.rfind('/');

66

13289391

  if (index == std::string::npos || index == full_path.size()) {

67

    return full_path;

68

69

13289391

  return full_path.substr(index + 1, full_path.size());

70

13289391

71

72

// Logs a warning for use of a deprecated field or runtime-overridden use of an

73

// otherwise fatal field. Throws a warning on use of a fatal by default field.

74

void deprecatedFieldHelper(Runtime::Loader* runtime, bool proto_annotated_as_deprecated,

75

                           bool proto_annotated_as_disallowed, const std::string& feature_name,

76

                           std::string error, const Protobuf::Message& message,

77

290

                           ProtobufMessage::ValidationVisitor& validation_visitor) {

78

// This option is for Envoy builds with --define deprecated_features=disabled

79

// The build options CI then verifies that as Envoy developers deprecate fields,

80

// that they update canonical configs and unit tests to not use those deprecated

81

// fields, by making their use fatal in the build options CI.

82

#ifdef ENVOY_DISABLE_DEPRECATED_FEATURES

83

  bool warn_only = false;

84

#else

85

290

  bool warn_only = true;

86

290

#endif

87

290

  if (runtime &&

88

290

      runtime->snapshot().getBoolean("envoy.features.fail_on_any_deprecated_feature", false)) {

89

1

    warn_only = false;

90

1

91

290

  bool warn_default = warn_only;

92

  // Allow runtime to be null both to not crash if this is called before server initialization,

93

  // and so proto validation works in context where runtime singleton is not set up (e.g.

94

  // standalone config validation utilities)

95

290

  if (runtime && proto_annotated_as_deprecated) {

96

    // This is set here, rather than above, so that in the absence of a

97

    // registry (i.e. test) the default for if a feature is allowed or not is

98

    // based on ENVOY_DISABLE_DEPRECATED_FEATURES.

99

198

    warn_only &= !proto_annotated_as_disallowed;

100

198

    warn_default = warn_only;

101

198

    warn_only = runtime->snapshot().deprecatedFeatureEnabled(feature_name, warn_only);

102

198

103

  // Note this only checks if the runtime override has an actual effect. It

104

  // does not change the logged warning if someone "allows" a deprecated but not

105

  // yet fatal field.

106

290

  const bool runtime_overridden = (warn_default == false && warn_only == true);

107

108

290

  std::string with_overridden = fmt::format(

109

290

      fmt::runtime(error),

110

290

      (runtime_overridden ? "runtime overrides to continue using now fatal-by-default " : ""));

111

112

290

  THROW_IF_NOT_OK(validation_visitor.onDeprecatedField(

113

290

      absl::StrCat("type ", message.GetTypeName(), " ", with_overridden), warn_only));

114

290

115

116

} // namespace

117

118

namespace ProtobufPercentHelper {

119

120

34002

uint64_t checkAndReturnDefault(uint64_t default_value, uint64_t max_value) {

121

34002

  ASSERT(default_value <= max_value);

122

34002

  return default_value;

123

34002

124

125

89

uint64_t convertPercent(double percent, uint64_t max_value) {

126

  // Checked by schema.

127

89

  ASSERT(percent >= 0.0 && percent <= 100.0);

128

89

  return max_value * (percent / 100.0);

129

89

130

131

1011097

bool evaluateFractionalPercent(envoy::type::v3::FractionalPercent percent, uint64_t random_value) {

132

1011097

  return random_value % fractionalPercentDenominatorToInt(percent.denominator()) <

133

1011097

         percent.numerator();

134

1011097

135

136

uint64_t fractionalPercentDenominatorToInt(

137

2022458

    const envoy::type::v3::FractionalPercent::DenominatorType& denominator) {

138

2022458

  switch (denominator) {

139

    PANIC_ON_PROTO_ENUM_SENTINEL_VALUES;

140

21235

  case envoy::type::v3::FractionalPercent::HUNDRED:

141

21235

    return 100;

142

2000191

  case envoy::type::v3::FractionalPercent::TEN_THOUSAND:

143

2000191

    return 10000;

144

1032

  case envoy::type::v3::FractionalPercent::MILLION:

145

1032

    return 1000000;

146

2022458

147

  PANIC_DUE_TO_CORRUPT_ENUM

148

149

150

} // namespace ProtobufPercentHelper

151

152

void ProtoExceptionUtil::throwMissingFieldException(const std::string& field_name,

153

4

                                                    const Protobuf::Message& message) {

154

4

  std::string error =

155

4

      fmt::format("Field '{}' is missing in: {}", field_name, message.DebugString());

156

4

  throwEnvoyExceptionOrPanic(error);

157

4

158

159

void ProtoExceptionUtil::throwProtoValidationException(const std::string& validation_error,

160

215

                                                       const Protobuf::Message& message) {

161

215

  std::string error = fmt::format("{}: Proto constraint validation failed ({})",

162

215

                                  message.DebugString(), validation_error);

163

215

  throwEnvoyExceptionOrPanic(error);

164

215

165

166

40163

size_t MessageUtil::hash(const Protobuf::Message& message) {

167

40163

#if defined(ENVOY_ENABLE_FULL_PROTOS)

168

40163

  return DeterministicProtoHash::hash(message);

169

#else

170

  return HashUtil::xxHash64(message.SerializeAsString());

171

#endif

172

40163

173

174

#if !defined(ENVOY_ENABLE_FULL_PROTOS)

175

// NOLINTNEXTLINE(readability-identifier-naming)

176

bool MessageLiteDifferencer::Equals(const Protobuf::Message& message1,

177

                                    const Protobuf::Message& message2) {

178

  return MessageUtil::hash(message1) == MessageUtil::hash(message2);

179

180

181

// NOLINTNEXTLINE(readability-identifier-naming)

182

bool MessageLiteDifferencer::Equivalent(const Protobuf::Message& message1,

183

                                        const Protobuf::Message& message2) {

184

  return Equals(message1, message2);

185

186

#endif

187

188

namespace {

189

190

void checkForDeprecatedNonRepeatedEnumValue(

191

    const Protobuf::Message& message, absl::string_view filename,

192

    const Protobuf::FieldDescriptor* field, const Protobuf::Reflection* reflection,

193

13289391

    Runtime::Loader* runtime, ProtobufMessage::ValidationVisitor& validation_visitor) {

194

  // Repeated fields will be handled by recursion in checkForUnexpectedFields.

195

13289391

  if (field->is_repeated() || field->cpp_type() != Protobuf::FieldDescriptor::CPPTYPE_ENUM) {

196

12715400

    return;

197

12715400

198

199

573991

  Protobuf::ReflectableMessage reflectable_message = createReflectableMessage(message);

200

573991

  bool default_value = !reflection->HasField(*reflectable_message, field);

201

202

573991

  const Protobuf::EnumValueDescriptor* enum_value_descriptor =

203

573991

      reflection->GetEnum(*reflectable_message, field);

204

573991

  if (!enum_value_descriptor->options().deprecated()) {

205

573978

    return;

206

573978

207

208

13

  const std::string error =

209

13

      absl::StrCat("Using {}", (default_value ? "the default now-" : ""), "deprecated value ",

210

13

                   enum_value_descriptor->name(), " for enum '", field->full_name(), "' from file ",

211

13

                   filename, ". This enum value will be removed from Envoy soon",

212

13

                   (default_value ? " so a non-default value must now be explicitly set" : ""),

213

13

                   ". Please see " ENVOY_DOC_URL_VERSION_HISTORY " for details.");

214

13

  deprecatedFieldHelper(

215

13

      runtime, true /*deprecated*/,

216

13

      enum_value_descriptor->options().GetExtension(envoy::annotations::disallowed_by_default_enum),

217

13

      absl::StrCat("envoy.deprecated_features:", enum_value_descriptor->full_name()), error,

218

13

      message, validation_visitor);

219

13

220

221

constexpr absl::string_view WipWarning =

222

    "API features marked as work-in-progress are not considered stable, are not covered by the "

223

    "threat model, are not supported by the security team, and are subject to breaking changes. Do "

224

    "not use this feature without understanding each of the previous points.";

225

226

class UnexpectedFieldProtoVisitor : public ProtobufMessage::ConstProtoVisitor {

227

public:

228

  UnexpectedFieldProtoVisitor(ProtobufMessage::ValidationVisitor& validation_visitor,

229

                              Runtime::Loader* runtime)

230

157988

      : validation_visitor_(validation_visitor), runtime_(runtime) {}

231

232

13289391

  void onField(const Protobuf::Message& message, const Protobuf::FieldDescriptor& field) override {

233

13289391

    Protobuf::ReflectableMessage reflectable_message = createReflectableMessage(message);

234

13289391

    const Protobuf::Reflection* reflection = reflectable_message->GetReflection();

235

13289391

    absl::string_view filename = filenameFromPath(field.file()->name());

236

237

    // Before we check to see if the field is in use, see if there's a

238

    // deprecated default enum value.

239

13289391

    checkForDeprecatedNonRepeatedEnumValue(message, filename, &field, reflection, runtime_,

240

13289391

                                           validation_visitor_);

241

242

    // If this field is not in use, continue.

243

13289391

    if ((field.is_repeated() && reflection->FieldSize(*reflectable_message, &field) == 0) ||

244

13289391

        (!field.is_repeated() && !reflection->HasField(*reflectable_message, &field))) {

245

10517709

      return;

246

10517709

247

248

2771682

    const auto& field_status = field.options().GetExtension(xds::annotations::v3::field_status);

249

2771682

    if (field_status.work_in_progress()) {

250

529

      validation_visitor_.onWorkInProgress(fmt::format(

251

529

          "field '{}' is marked as work-in-progress. {}", field.full_name(), WipWarning));

252

529

253

254

    // If this field is deprecated, warn or throw an error.

255

2771682

    if (field.options().deprecated()) {

256

277

      const std::string warning =

257

277

          absl::StrCat("Using {}deprecated option '", field.full_name(), "' from file ", filename,

258

277

                       ". This configuration will be removed from "

259

277

                       "Envoy soon. Please see " ENVOY_DOC_URL_VERSION_HISTORY " for details.");

260

261

277

      deprecatedFieldHelper(runtime_, true /*deprecated*/,

262

277

                            field.options().GetExtension(envoy::annotations::disallowed_by_default),

263

277

                            absl::StrCat("envoy.deprecated_features:", field.full_name()), warning,

264

277

                            message, validation_visitor_);

265

277

266

2771682

267

268

  absl::Status onMessage(const Protobuf::Message& message,

269

1716946

                         absl::Span<const Protobuf::Message* const> parents, bool) override {

270

1716946

    Protobuf::ReflectableMessage reflectable_message = createReflectableMessage(message);

271

1716946

    if (reflectable_message->GetDescriptor()

272

1716946

            ->options()

273

1716946

            .GetExtension(xds::annotations::v3::message_status)

274

1716946

            .work_in_progress()) {

275

321

      validation_visitor_.onWorkInProgress(fmt::format(

276

321

          "message '{}' is marked as work-in-progress. {}", message.GetTypeName(), WipWarning));

277

321

278

279

1716946

    const auto& udpa_file_options =

280

1716946

        reflectable_message->GetDescriptor()->file()->options().GetExtension(

281

1716946

            udpa::annotations::file_status);

282

1716946

    const auto& xds_file_options =

283

1716946

        reflectable_message->GetDescriptor()->file()->options().GetExtension(

284

1716946

            xds::annotations::v3::file_status);

285

1716946

    if (udpa_file_options.work_in_progress() || xds_file_options.work_in_progress()) {

286

5888

      validation_visitor_.onWorkInProgress(fmt::format(

287

5888

          "message '{}' is contained in proto file '{}' marked as work-in-progress. {}",

288

5888

          message.GetTypeName(), reflectable_message->GetDescriptor()->file()->name(), WipWarning));

289

5888

290

291

    // Reject unknown fields.

292

1716946

    const auto& unknown_fields =

293

1716946

        reflectable_message->GetReflection()->GetUnknownFields(*reflectable_message);

294

1716946

    if (!unknown_fields.empty()) {

295

7

      std::string error_msg;

296

15

      for (int n = 0; n < unknown_fields.field_count(); ++n) {

297

8

        absl::StrAppend(&error_msg, n > 0 ? ", " : "", unknown_fields.field(n).number());

298

8

299

7

      if (!error_msg.empty()) {

300

7

        RETURN_IF_NOT_OK(validation_visitor_.onUnknownField(

301

7

            fmt::format("type {}({}) with unknown field set {{{}}}", message.GetTypeName(),

302

7

                        !parents.empty()

303

7

                            ? absl::StrJoin(parents, "::",

304

7

                                            [](std::string* out, const Protobuf::Message* const m) {

305

7

                                              absl::StrAppend(out, m->GetTypeName());

306

7

})

307

7

                            : "root",

308

7

                        error_msg)));

309

310

7

311

1716939

    return absl::OkStatus();

312

1716946

313

314

private:

315

  ProtobufMessage::ValidationVisitor& validation_visitor_;

316

  Runtime::Loader* runtime_;

317

};

318

319

} // namespace

320

321

void MessageUtil::checkForUnexpectedFields(const Protobuf::Message& message,

322

                                           ProtobufMessage::ValidationVisitor& validation_visitor,

323

157988

                                           bool recurse_into_any) {

324

157988

  Runtime::Loader* runtime = validation_visitor.runtime().has_value()

325

157988

                                 ? &validation_visitor.runtime().value().get()

326

157988

                                 : nullptr;

327

157988

  UnexpectedFieldProtoVisitor unexpected_field_visitor(validation_visitor, runtime);

328

157988

  THROW_IF_NOT_OK(

329

157988

      ProtobufMessage::traverseMessage(unexpected_field_visitor, message, recurse_into_any));

330

157988

331

332

namespace {

333

334

// A proto visitor that validates the correctness of google.protobuf.Duration messages

335

// as defined by Envoy's duration constraints.

336

class DurationFieldProtoVisitor : public ProtobufMessage::ConstProtoVisitor {

337

public:

338

10312614

  void onField(const Protobuf::Message&, const Protobuf::FieldDescriptor&) override {}

339

340

  absl::Status onMessage(const Protobuf::Message& message,

341

1269395

                         absl::Span<const Protobuf::Message* const>, bool) override {

342

1269395

    const Protobuf::ReflectableMessage reflectable_message = createReflectableMessage(message);

343

1269395

    if (reflectable_message->GetDescriptor()->full_name() == "google.protobuf.Duration") {

344

43571

      Protobuf::Duration duration_message;

345

43571

#if defined(ENVOY_ENABLE_FULL_PROTOS)

346

43571

      duration_message.CheckTypeAndMergeFrom(message);

347

#else

348

      duration_message.MergeFromCord(message.SerializeAsCord());

349

#endif

350

      // Validate the value of the duration.

351

43571

      RETURN_IF_NOT_OK(validateDurationNoThrow(duration_message));

352

43566

353

1269390

    return absl::OkStatus();

354

1269395

355

};

356

357

} // namespace

358

359

147494

void MessageUtil::validateDurationFields(const Protobuf::Message& message, bool recurse_into_any) {

360

147494

  DurationFieldProtoVisitor duration_field_visitor;

361

147494

  THROW_IF_NOT_OK(

362

147494

      ProtobufMessage::traverseMessage(duration_field_visitor, message, recurse_into_any));

363

147494

364

365

namespace {

366

367

class PgvCheckVisitor : public ProtobufMessage::ConstProtoVisitor {

368

public:

369

  absl::Status onMessage(const Protobuf::Message& message,

370

                         absl::Span<const Protobuf::Message* const>,

371

89

                         bool was_any_or_top_level) override {

372

89

    Protobuf::ReflectableMessage reflectable_message = createReflectableMessage(message);

373

89

    std::string err;

374

    // PGV verification is itself recursive up to the point at which it hits an Any message. As

375

    // such, to avoid N^2 checking of the tree, we only perform an additional check at the point

376

    // at which PGV would have stopped because it does not itself check within Any messages.

377

89

    if (was_any_or_top_level &&

378

89

        !pgv::BaseValidator::AbstractCheckMessage(*reflectable_message, &err)) {

379

2

      std::string error = fmt::format("{}: Proto constraint validation failed ({})",

380

2

                                      reflectable_message->DebugString(), err);

381

2

      return absl::InvalidArgumentError(error);

382

2

383

87

    return absl::OkStatus();

384

89

385

386

696

  void onField(const Protobuf::Message&, const Protobuf::FieldDescriptor&) override {}

387

};

388

389

} // namespace

390

391

7

void MessageUtil::recursivePgvCheck(const Protobuf::Message& message) {

392

7

  PgvCheckVisitor visitor;

393

7

  THROW_IF_NOT_OK(ProtobufMessage::traverseMessage(visitor, message, true));

394

7

395

396

112

void MessageUtil::packFrom(Protobuf::Any& any_message, const Protobuf::Message& message) {

397

112

#if defined(ENVOY_ENABLE_FULL_PROTOS)

398

112

  any_message.PackFrom(message);

399

#else

400

  any_message.set_type_url(message.GetTypeName());

401

  any_message.set_value(message.SerializeAsString());

402

#endif

403

112

404

405

120448

absl::Status MessageUtil::unpackTo(const Protobuf::Any& any_message, Protobuf::Message& message) {

406

120448

#if defined(ENVOY_ENABLE_FULL_PROTOS)

407

120448

  if (!any_message.UnpackTo(&message)) {

408

17

    return absl::InternalError(absl::StrCat("Unable to unpack as ",

409

17

                                            message.GetDescriptor()->full_name(), ": ",

410

17

                                            any_message.DebugString()));

411

#else

412

  if (!message.ParseFromString(any_message.value())) {

413

    return absl::InternalError(

414

        absl::StrCat("Unable to unpack as ", message.GetTypeName(), ": ", any_message.type_url()));

415

#endif

416

17

417

  // Ok Status is returned if `UnpackTo` succeeded.

418

120431

  return absl::OkStatus();

419

120448

420

421

std::string MessageUtil::convertToStringForLogs(const Protobuf::Message& message, bool pretty_print,

422

112

                                                bool always_print_primitive_fields) {

423

112

#ifdef ENVOY_ENABLE_YAML

424

112

  return getJsonStringFromMessageOrError(message, pretty_print, always_print_primitive_fields);

425

#else

426

  UNREFERENCED_PARAMETER(pretty_print);

427

  UNREFERENCED_PARAMETER(always_print_primitive_fields);

428

  return message.DebugString();

429

#endif

430

112

431

432

48

Protobuf::Struct MessageUtil::keyValueStruct(const std::string& key, const std::string& value) {

433

48

  Protobuf::Struct struct_obj;

434

48

  Protobuf::Value val;

435

48

  val.set_string_value(value);

436

48

  (*struct_obj.mutable_fields())[key] = val;

437

48

  return struct_obj;

438

48

439

440

478

Protobuf::Struct MessageUtil::keyValueStruct(const std::map<std::string, std::string>& fields) {

441

478

  Protobuf::Struct struct_obj;

442

478

  Protobuf::Value val;

443

2369

  for (const auto& pair : fields) {

444

2369

    val.set_string_value(pair.second);

445

2369

    (*struct_obj.mutable_fields())[pair.first] = val;

446

2369

447

478

  return struct_obj;

448

478

449

450

38

std::string MessageUtil::codeEnumToString(absl::StatusCode code) {

451

38

  std::string result = absl::StatusCodeToString(code);

452

  // This preserves the behavior of the `ProtobufUtil::Status(code, "").ToString();`

453

38

  return !result.empty() ? result : "UNKNOWN: ";

454

38

455

456

namespace {

457

458

// Forward declaration for mutually-recursive helper functions.

459

void redact(Protobuf::Message* message, bool ancestor_is_sensitive);

460

461

using Transform = std::function<void(Protobuf::Message*, const Protobuf::Reflection*,

462

                                     const Protobuf::FieldDescriptor*)>;

463

464

// To redact opaque types, namely `Any` and `TypedStruct`, we have to reify them to the concrete

465

// message types specified by their `type_url` before we can redact their contents. This is mostly

466

// identical between `Any` and `TypedStruct`, the only difference being how they are packed and

467

// unpacked. Note that we have to use reflection on the opaque type here, rather than downcasting

468

// to `Any` or `TypedStruct`, because any message we might be handling could have originated from

469

// a `DynamicMessageFactory`.

470

bool redactOpaque(Protobuf::Message* message, bool ancestor_is_sensitive,

471

31729

                  absl::string_view opaque_type_name, Transform unpack, Transform repack) {

472

  // Ensure this message has the opaque type we're expecting.

473

31729

  Protobuf::ReflectableMessage reflectable_message = createReflectableMessage(*message);

474

31729

  const auto* opaque_descriptor = reflectable_message->GetDescriptor();

475

31729

  if (opaque_descriptor->full_name() != opaque_type_name) {

476

30868

    return false;

477

30868

478

479

  // Find descriptors for the `type_url` and `value` fields. The `type_url` field must not be

480

  // empty, but `value` may be (in which case our work is done).

481

861

  const auto* reflection = reflectable_message->GetReflection();

482

861

  const auto* type_url_field_descriptor = opaque_descriptor->FindFieldByName("type_url");

483

861

  const auto* value_field_descriptor = opaque_descriptor->FindFieldByName("value");

484

861

  ASSERT(type_url_field_descriptor != nullptr && value_field_descriptor != nullptr);

485

861

  if (!reflection->HasField(*reflectable_message, type_url_field_descriptor) &&

486

861

      !reflection->HasField(*reflectable_message, value_field_descriptor)) {

487

3

    return true;

488

3

489

858

  if (!reflection->HasField(*reflectable_message, type_url_field_descriptor) ||

490

858

      !reflection->HasField(*reflectable_message, value_field_descriptor)) {

491

64

    return false;

492

64

493

494

  // Try to find a descriptor for `type_url` in the pool and instantiate a new message of the

495

  // correct concrete type.

496

794

  const std::string type_url(

497

794

      reflection->GetString(*reflectable_message, type_url_field_descriptor));

498

794

  const std::string concrete_type_name(TypeUtil::typeUrlToDescriptorFullName(type_url));

499

794

  const auto* concrete_descriptor =

500

794

      Protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName(concrete_type_name);

501

794

  if (concrete_descriptor == nullptr) {

502

    // If the type URL doesn't correspond to a known proto, don't try to reify it, just treat it

503

    // like any other message. See the documented limitation on `MessageUtil::redact()` for more

504

    // context.

505

3

    ENVOY_LOG_MISC(warn, "Could not reify {} with unknown type URL {}", opaque_type_name, type_url);

506

3

    return false;

507

3

508

791

  Protobuf::DynamicMessageFactory message_factory;

509

791

  std::unique_ptr<Protobuf::Message> typed_message(

510

791

      message_factory.GetPrototype(concrete_descriptor)->New());

511

512

  // Finally we can unpack, redact, and repack the opaque message using the provided callbacks.

513

514

  // Note: the content of opaque types may contain illegal content that mismatches the type_url

515

  // which may cause unpacking to fail. We catch the exception here to avoid crashing Envoy.

516

791

  TRY_ASSERT_MAIN_THREAD { unpack(typed_message.get(), reflection, value_field_descriptor); }

517

791

  END_TRY CATCH(const EnvoyException& e, {

518

791

    ENVOY_LOG_MISC(warn, "Could not unpack {} with type URL {}: {}", opaque_type_name, type_url,

519

791

                   e.what());

520

791

    return false;

521

791

});

522

789

  redact(typed_message.get(), ancestor_is_sensitive);

523

789

  repack(typed_message.get(), reflection, value_field_descriptor);

524

789

  return true;

525

791

526

527

11100

bool redactAny(Protobuf::Message* message, bool ancestor_is_sensitive) {

528

11100

  return redactOpaque(

529

11100

      message, ancestor_is_sensitive, "google.protobuf.Any",

530

11100

      [message](Protobuf::Message* typed_message, const Protobuf::Reflection* reflection,

531

11100

                const Protobuf::FieldDescriptor* field_descriptor) {

532

779

        Protobuf::ReflectableMessage reflectable_message = createReflectableMessage(*message);

533

        // To unpack an `Any`, parse the serialized proto.

534

779

        typed_message->ParseFromString(

535

779

            reflection->GetString(*reflectable_message, field_descriptor));

536

779

},

537

11100

      [message](Protobuf::Message* typed_message, const Protobuf::Reflection* reflection,

538

11100

                const Protobuf::FieldDescriptor* field_descriptor) {

539

779

        Protobuf::ReflectableMessage reflectable_message = createReflectableMessage(*message);

540

        // To repack an `Any`, reserialize its proto.

541

779

        reflection->SetString(&(*reflectable_message), field_descriptor,

542

779

                              typed_message->SerializeAsString());

543

779

});

544

11100

545

546

// To redact a `TypedStruct`, we have to reify it based on its `type_url` to redact it.

547

bool redactTypedStruct(Protobuf::Message* message, const char* typed_struct_type,

548

20629

                       bool ancestor_is_sensitive) {

549

20629

  return redactOpaque(

550

20629

      message, ancestor_is_sensitive, typed_struct_type,

551

20629

      [message](Protobuf::Message* typed_message, const Protobuf::Reflection* reflection,

552

20629

                const Protobuf::FieldDescriptor* field_descriptor) {

553

12

#ifdef ENVOY_ENABLE_YAML

554

        // To unpack a `TypedStruct`, convert the struct from JSON.

555

12

        MessageUtil::jsonConvert(reflection->GetMessage(*message, field_descriptor),

556

12

                                 *typed_message);

557

#else

558

        UNREFERENCED_PARAMETER(message);

559

        UNREFERENCED_PARAMETER(typed_message);

560

        UNREFERENCED_PARAMETER(reflection);

561

        UNREFERENCED_PARAMETER(field_descriptor);

562

        IS_ENVOY_BUG("redaction requested with JSON/YAML support removed");

563

#endif

564

12

},

565

20629

      [message](Protobuf::Message* typed_message, const Protobuf::Reflection* reflection,

566

20629

                const Protobuf::FieldDescriptor* field_descriptor) {

567

  // To repack a `TypedStruct`, convert the message back to JSON.

568

10

#ifdef ENVOY_ENABLE_YAML

569

10

        MessageUtil::jsonConvert(*typed_message,

570

10

                                 *(reflection->MutableMessage(message, field_descriptor)));

571

#else

572

        UNREFERENCED_PARAMETER(message);

573

        UNREFERENCED_PARAMETER(typed_message);

574

        UNREFERENCED_PARAMETER(reflection);

575

        UNREFERENCED_PARAMETER(field_descriptor);

576

        IS_ENVOY_BUG("redaction requested with JSON/YAML support removed");

577

#endif

578

10

});

579

20629

580

581

// Recursive helper method for MessageUtil::redact() below.

582

11100

void redact(Protobuf::Message* message, bool ancestor_is_sensitive) {

583

11100

  if (redactAny(message, ancestor_is_sensitive) ||

584

11100

      redactTypedStruct(message, "xds.type.v3.TypedStruct", ancestor_is_sensitive) ||

585

11100

      redactTypedStruct(message, "udpa.type.v1.TypedStruct", ancestor_is_sensitive)) {

586

792

    return;

587

792

588

589

10308

  Protobuf::ReflectableMessage reflectable_message = createReflectableMessage(*message);

590

10308

  const auto* descriptor = reflectable_message->GetDescriptor();

591

10308

  const auto* reflection = reflectable_message->GetReflection();

592

82166

  for (int i = 0; i < descriptor->field_count(); ++i) {

593

71858

    const auto* field_descriptor = descriptor->field(i);

594

595

    // Redact if this field or any of its ancestors have the `sensitive` option set.

596

71858

    const bool sensitive = ancestor_is_sensitive ||

597

71858

                           field_descriptor->options().GetExtension(udpa::annotations::sensitive);

598

599

71858

    if (field_descriptor->type() == Protobuf::FieldDescriptor::TYPE_MESSAGE) {

600

      // Recursive case: traverse message fields.

601

33110

      if (field_descriptor->is_map()) {

602

        // Redact values of maps only. Redacting both leaves the map with multiple "[redacted]"

603

        // keys.

604

768

        const int field_size = reflection->FieldSize(*reflectable_message, field_descriptor);

605

1144

        for (int i = 0; i < field_size; ++i) {

606

376

          Protobuf::Message* map_pair_base =

607

376

              reflection->MutableRepeatedMessage(&(*reflectable_message), field_descriptor, i);

608

376

          Protobuf::ReflectableMessage map_pair = createReflectableMessage(*map_pair_base);

609

376

          auto* value_field_desc = map_pair->GetDescriptor()->FindFieldByName("value");

610

376

          if (sensitive && (value_field_desc->type() == Protobuf::FieldDescriptor::TYPE_STRING ||

611

2

                            value_field_desc->type() == Protobuf::FieldDescriptor::TYPE_BYTES)) {

612

1

            map_pair->GetReflection()->SetString(&(*map_pair), value_field_desc, "[redacted]");

613

375

          } else if (value_field_desc->type() == Protobuf::FieldDescriptor::TYPE_MESSAGE) {

614

372

            redact(map_pair->GetReflection()->MutableMessage(&(*map_pair), value_field_desc),

615

372

                   sensitive);

616

372

          } else if (sensitive) {

617

1

            map_pair->GetReflection()->ClearField(&(*map_pair), value_field_desc);

618

1

619

376

620

32342

      } else if (field_descriptor->is_repeated()) {

621

4578

        const int field_size = reflection->FieldSize(*reflectable_message, field_descriptor);

622

10913

        for (int i = 0; i < field_size; ++i) {

623

6335

          redact(reflection->MutableRepeatedMessage(&(*reflectable_message), field_descriptor, i),

624

6335

                 sensitive);

625

6335

626

27767

      } else if (reflection->HasField(*reflectable_message, field_descriptor)) {

627

3458

        redact(reflection->MutableMessage(&(*reflectable_message), field_descriptor), sensitive);

628

3458

629

39056

    } else if (sensitive) {

630

      // Base case: replace strings and bytes with "[redacted]" and clear all others.

631

1034

      if (field_descriptor->type() == Protobuf::FieldDescriptor::TYPE_STRING ||

632

1034

          field_descriptor->type() == Protobuf::FieldDescriptor::TYPE_BYTES) {

633

948

        if (field_descriptor->is_repeated()) {

634

86

          const int field_size = reflection->FieldSize(*reflectable_message, field_descriptor);

635

120

          for (int i = 0; i < field_size; ++i) {

636

34

            reflection->SetRepeatedString(&(*reflectable_message), field_descriptor, i,

637

34

                                          "[redacted]");

638

34

639

862

        } else if (reflection->HasField(*reflectable_message, field_descriptor)) {

640

215

          reflection->SetString(&(*reflectable_message), field_descriptor, "[redacted]");

641

215

642

948

      } else {

643

86

        reflection->ClearField(&(*reflectable_message), field_descriptor);

644

86

645

1034

646

71858

647

10308

648

649

} // namespace

650

651

146

void MessageUtil::redact(Protobuf::Message& message) {

652

146

  ::Envoy::redact(&message, /* ancestor_is_sensitive = */ false);

653

146

654

655

359

std::string MessageUtil::toTextProto(const Protobuf::Message& message) {

656

359

#if defined(ENVOY_ENABLE_FULL_PROTOS)

657

359

  std::string text_format;

658

359

  Protobuf::TextFormat::Printer printer;

659

359

  printer.SetExpandAny(true);

660

359

  printer.SetHideUnknownFields(true);

661

359

  bool result = printer.PrintToString(message, &text_format);

662

359

  ASSERT(result);

663

359

  return text_format;

664

#else

665

  // Note that MessageLite::DebugString never had guarantees of producing

666

  // serializable text proto representation.

667

  return message.DebugString();

668

#endif

669

359

670

671

2173

bool ValueUtil::equal(const Protobuf::Value& v1, const Protobuf::Value& v2) {

672

2173

  Protobuf::Value::KindCase kind = v1.kind_case();

673

2173

  if (kind != v2.kind_case()) {

674

3

    return false;

675

3

676

677

2170

  switch (kind) {

678

  case Protobuf::Value::KIND_NOT_SET:

679

    return v2.kind_case() == Protobuf::Value::KIND_NOT_SET;

680

681

1

  case Protobuf::Value::kNullValue:

682

1

    return true;

683

684

7

  case Protobuf::Value::kNumberValue:

685

7

    return v1.number_value() == v2.number_value();

686

687

2148

  case Protobuf::Value::kStringValue:

688

2148

    return v1.string_value() == v2.string_value();

689

690

7

  case Protobuf::Value::kBoolValue:

691

7

    return v1.bool_value() == v2.bool_value();

692

693

4

  case Protobuf::Value::kStructValue: {

694

4

    const Protobuf::Struct& s1 = v1.struct_value();

695

4

    const Protobuf::Struct& s2 = v2.struct_value();

696

4

    if (s1.fields_size() != s2.fields_size()) {

697

1

      return false;

698

1

699

5

    for (const auto& it1 : s1.fields()) {

700

5

      const auto& it2 = s2.fields().find(it1.first);

701

5

      if (it2 == s2.fields().end()) {

702

        return false;

703

704

705

5

      if (!equal(it1.second, it2->second)) {

706

1

        return false;

707

1

708

5

709

2

    return true;

710

3

711

712

3

  case Protobuf::Value::kListValue: {

713

3

    const Protobuf::ListValue& l1 = v1.list_value();

714

3

    const Protobuf::ListValue& l2 = v2.list_value();

715

3

    if (l1.values_size() != l2.values_size()) {

716

1

      return false;

717

1

718

4

    for (int i = 0; i < l1.values_size(); i++) {

719

3

      if (!equal(l1.values(i), l2.values(i))) {

720

1

        return false;

721

1

722

3

723

1

    return true;

724

2

725

2170

726

  return false;

727

2170

728

729

2042

const Protobuf::Value& ValueUtil::nullValue() {

730

2042

  static const auto* v = []() -> Protobuf::Value* {

731

29

    auto* vv = new Protobuf::Value();

732

29

    vv->set_null_value(Protobuf::NULL_VALUE);

733

29

    return vv;

734

29

  }();

735

2042

  return *v;

736

2042

737

738

1122235

Protobuf::Value ValueUtil::stringValue(absl::string_view str) {

739

1122235

  Protobuf::Value val;

740

1122235

  val.set_string_value(str);

741

1122235

  return val;

742

1122235

743

744

167

Protobuf::Value ValueUtil::optionalStringValue(const absl::optional<std::string>& str) {

745

167

  if (str.has_value()) {

746

136

    return ValueUtil::stringValue(str.value());

747

136

748

31

  return ValueUtil::nullValue();

749

167

750

751

32668

Protobuf::Value ValueUtil::boolValue(bool b) {

752

32668

  Protobuf::Value val;

753

32668

  val.set_bool_value(b);

754

32668

  return val;

755

32668

756

757

822

Protobuf::Value ValueUtil::structValue(const Protobuf::Struct& obj) {

758

822

  Protobuf::Value val;

759

822

  (*val.mutable_struct_value()) = obj;

760

822

  return val;

761

822

762

763

698

Protobuf::Value ValueUtil::listValue(const std::vector<Protobuf::Value>& values) {

764

698

  auto list = std::make_unique<Protobuf::ListValue>();

765

1954

  for (const auto& value : values) {

766

1954

    *list->add_values() = value;

767

1954

768

698

  Protobuf::Value val;

769

698

  val.set_allocated_list_value(list.release());

770

698

  return val;

771

698

772

773

50757

uint64_t DurationUtil::durationToMilliseconds(const Protobuf::Duration& duration) {

774

50757

  validateDuration(duration);

775

50757

  return Protobuf::util::TimeUtil::DurationToMilliseconds(duration);

776

50757

777

778

absl::StatusOr<uint64_t>

779

26

DurationUtil::durationToMillisecondsNoThrow(const Protobuf::Duration& duration) {

780

26

  const absl::Status result = validateDurationNoThrow(duration);

781

26

  if (!result.ok()) {

782

8

    return result;

783

8

784

18

  return Protobuf::util::TimeUtil::DurationToMilliseconds(duration);

785

26

786

787

109

uint64_t DurationUtil::durationToSeconds(const Protobuf::Duration& duration) {

788

109

  validateDuration(duration);

789

109

  return Protobuf::util::TimeUtil::DurationToSeconds(duration);

790

109

791

792

void TimestampUtil::systemClockToTimestamp(const SystemTime system_clock_time,

793

849

                                           Protobuf::Timestamp& timestamp) {

794

  // Converts to millisecond-precision Timestamp by explicitly casting to millisecond-precision

795

  // time_point.

796

849

  timestamp.MergeFrom(Protobuf::util::TimeUtil::MillisecondsToTimestamp(

797

849

      std::chrono::time_point_cast<std::chrono::milliseconds>(system_clock_time)

798

849

          .time_since_epoch()

799

849

          .count()));

800

849

801

802

200714

absl::string_view TypeUtil::typeUrlToDescriptorFullName(absl::string_view type_url) {

803

200714

  const size_t pos = type_url.rfind('/');

804

200714

  if (pos != absl::string_view::npos) {

805

200556

    type_url = type_url.substr(pos + 1);

806

200556

807

200714

  return type_url;

808

200714

809

810

353

std::string TypeUtil::descriptorFullNameToTypeUrl(absl::string_view type) {

811

353

  return "type.googleapis.com/" + std::string(type);

812

353

813

814

17

void StructUtil::update(Protobuf::Struct& obj, const Protobuf::Struct& with) {

815

17

  auto& obj_fields = *obj.mutable_fields();

816

817

21

  for (const auto& [key, val] : with.fields()) {

818

21

    auto& obj_key = obj_fields[key];

819

820

    // If the types are different, the last one wins.

821

21

    const auto val_kind = val.kind_case();

822

21

    if (val_kind != obj_key.kind_case()) {

823

6

      obj_key = val;

824

6

      continue;

825

6

826

827

    // Otherwise, the strategy depends on the value kind.

828

15

    switch (val.kind_case()) {

829

    // For scalars, the last one wins.

830

1

    case Protobuf::Value::kNullValue:

831

5

    case Protobuf::Value::kNumberValue:

832

7

    case Protobuf::Value::kStringValue:

833

8

    case Protobuf::Value::kBoolValue:

834

8

      obj_key = val;

835

8

      break;

836

    // If we got a structure, recursively update.

837

4

    case Protobuf::Value::kStructValue:

838

4

      update(*obj_key.mutable_struct_value(), val.struct_value());

839

4

      break;

840

    // For lists, append the new values.

841

3

    case Protobuf::Value::kListValue: {

842

3

      auto& obj_key_vec = *obj_key.mutable_list_value()->mutable_values();

843

3

      const auto& vals = val.list_value().values();

844

3

      obj_key_vec.MergeFrom(vals);

845

3

      break;

846

7

847

    case Protobuf::Value::KIND_NOT_SET:

848

      break;

849

15

850

15

851

17

852

853

absl::Status MessageUtil::loadFromFile(const std::string& path, Protobuf::Message& message,

854

                                       ProtobufMessage::ValidationVisitor& validation_visitor,

855

21373

                                       Api::Api& api) {

856

21373

  auto file_or_error = api.fileSystem().fileReadToEnd(path);

857

21373

  RETURN_IF_NOT_OK_REF(file_or_error.status());

858

21282

  const std::string contents = file_or_error.value();

859

  // If the filename ends with .pb, attempt to parse it as a binary proto.

860

21282

  if (absl::EndsWithIgnoreCase(path, FileExtensions::get().ProtoBinary)) {

861

    // Attempt to parse the binary format.

862

10550

    if (message.ParseFromString(contents)) {

863

10550

      MessageUtil::checkForUnexpectedFields(message, validation_visitor);

864

10550

865

    // Ideally this would return an error if ParseFromString fails for consistency

866

    // but instead it will silently fail.

867

10550

    return absl::OkStatus();

868

10550

869

870

  // If the filename ends with .pb_text, attempt to parse it as a text proto.

871

10732

  if (absl::EndsWithIgnoreCase(path, FileExtensions::get().ProtoText)) {

872

398

#if defined(ENVOY_ENABLE_FULL_PROTOS)

873

398

    if (Protobuf::TextFormat::ParseFromString(contents, &message)) {

874

397

      return absl::OkStatus();

875

397

876

1

#endif

877

1

    return absl::InvalidArgumentError(absl::StrCat("Unable to parse file \"", path,

878

1

                                                   "\" as a text protobuf (type ",

879

1

                                                   message.GetTypeName(), ")"));

880

398

881

10334

#ifdef ENVOY_ENABLE_YAML

882

10334

  if (absl::EndsWithIgnoreCase(path, FileExtensions::get().Yaml) ||

883

10334

      absl::EndsWithIgnoreCase(path, FileExtensions::get().Yml)) {

884

    // loadFromYaml throws an error if parsing fails.

885

570

    loadFromYaml(contents, message, validation_visitor);

886

10219

  } else {

887

    // loadFromJson does not consistently trow an error if parsing fails.

888

    // Ideally we would handle that case here.

889

9764

    loadFromJson(contents, message, validation_visitor);

890

9764

891

#else

892

  return absl::InvalidArgumentError(

893

      absl::StrCat("Unable to parse file \"", path, "\" (type ", message.GetTypeName(), ")"));

894

#endif

895

10334

  return absl::OkStatus();

896

10732

897

898

} // namespace Envoy