/src/mod_auth_openidc/src/metrics.c

Source
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

/***************************************************************************
 * Copyright (C) 2023-2026 ZmartZone Holding BV
 * All rights reserved.
 *
 * DISCLAIMER OF WARRANTIES:
 *
 * THE SOFTWARE PROVIDED HEREUNDER IS PROVIDED ON AN "AS IS" BASIS, WITHOUT
 * ANY WARRANTIES OR REPRESENTATIONS EXPRESS, IMPLIED OR STATUTORY; INCLUDING,
 * WITHOUT LIMITATION, WARRANTIES OF QUALITY, PERFORMANCE, NONINFRINGEMENT,
 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.  NOR ARE THERE ANY
 * WARRANTIES CREATED BY A COURSE OR DEALING, COURSE OF PERFORMANCE OR TRADE
 * USAGE.  FURTHERMORE, THERE ARE NO WARRANTIES THAT THE SOFTWARE WILL MEET
 * YOUR NEEDS OR BE FREE FROM ERRORS, OR THAT THE OPERATION OF THE SOFTWARE
 * WILL BE UNINTERRUPTED.  IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * @Author: Hans Zandbelt - hans.zandbelt@openidc.com
 */

// clang-format off

#ifndef WIN32
#include <unistd.h>
#endif

#include "util/util.h"
#include "http.h"
#include "cache/cache.h"
#include "cfg/cfg_int.h"
#include "metrics.h"
#include <limits.h>
#include <apr_atomic.h>
#include <apr_shm.h>
#include <apr_lib.h>

// NB: formatting matters for docs script from here until clang-format on

// KEEP THIS: start-of-classes

#define OM_CLASS_AUTH_TYPE     "authtype"       // Request counter, overall and per AuthType: openid-connect, oauth20 and auth-openidc.
#define OM_CLASS_AUTHN         "authn"          // Authentication request creation and response processing.
#define OM_CLASS_AUTHZ         "authz"          // Authorization errors per OIDCUnAutzAction (per Require statement, not overall).
#define OM_CLASS_REQUIRE_CLAIM "require.claim"  // Match/failure count of Require claim directives (per Require statement, not overall).
#define OM_CLASS_CLAIM         "claim"          // Claims per value
#define OM_CLASS_PROVIDER      "provider"       // Requests to the provider [token, userinfo, metadata] endpoints.
#define OM_CLASS_SESSION       "session"        // Existing session processing.
#define OM_CLASS_CACHE         "cache"          // Cache read/write timings and errors.
#define OM_CLASS_LOGOUT        "logout"         // Front- and back-channel logout processing.
#define OM_CLASS_REDIRECT_URI  "redirect_uri"   // Requests to the Redirect URI, per type.
#define OM_CLASS_CONTENT       "content"        // Requests to the content handler, per type of request: info, metrics, jwks, etc.

// KEEP THIS: end-of-classes

// NB: order must match the oidc_metrics_timing_type_t enum type in metrics.h

const oidc_metrics_timing_info_t _oidc_metrics_timings_info[] = {

  // KEEP THIS: start-of-timers

  { OM_CLASS_AUTH_TYPE, "handler", "the overall authn+authz processing time" },

  { OM_CLASS_AUTHN,    "request",  "authentication requests" },
  { OM_CLASS_AUTHN,    "response", "authentication responses" },

  { OM_CLASS_SESSION,  "valid",    "successfully validated existing sessions" },

  { OM_CLASS_PROVIDER, "metadata", "provider discovery document requests" },
  { OM_CLASS_PROVIDER, "token",    "provider token requests" },
  { OM_CLASS_PROVIDER, "refresh",  "provider refresh token requests" },
  { OM_CLASS_PROVIDER, "userinfo", "provider userinfo requests" },
  { OM_CLASS_PROVIDER, "jwks",     "provider JWKs document requests" },
  { OM_CLASS_PROVIDER, "par",      "provider pushed authorization requests" },

  { OM_CLASS_CACHE,    "read",     "cache read requests" },
  { OM_CLASS_CACHE,    "write",    "cache write requests" },

  // KEEP THIS: end-of-timers

};

// NB: order must match the oidc_metrics_counter_type_t enum type in metrics.h

const oidc_metrics_counter_info_t _oidc_metrics_counters_info[] = {
#define OIDC_METRICS_COUNTER_INFO(id, class, name, desc) {class, name, desc},
    OIDC_METRICS_COUNTERS_LIST(OIDC_METRICS_COUNTER_INFO)
#undef OIDC_METRICS_COUNTER_INFO
};

// clang-format on

/* one entry per oidc_metrics_counter_type_t value */
OIDC_STATIC_ASSERT(sizeof(_oidc_metrics_counters_info) / sizeof(oidc_metrics_counter_info_t) == OM_NUMBER_OF_COUNTERS,
       counters_info_matches_enum);
/* one entry per oidc_metrics_timing_type_t value */
OIDC_STATIC_ASSERT(sizeof(_oidc_metrics_timings_info) / sizeof(oidc_metrics_timing_info_t) == OM_NUMBER_OF_TIMINGS,
       timings_info_matches_enum);

typedef struct oidc_metrics_t {
  apr_pool_t *pool;
  apr_hash_t *counters;
  apr_hash_t *timings;
} oidc_metrics_t;

// pointer to the shared memory segment that holds the JSON metrics data
static apr_shm_t *_oidc_metrics_cache = NULL;
// flag to record if we are a parent process or a child process
static apr_byte_t _oidc_metrics_is_parent = FALSE;
/* Atomic stop flag shared by the flush, cleanup, and child-init paths. */
static volatile apr_uint32_t _oidc_metrics_thread_exit = 0;
// mutex to protect the shared memory storage
static oidc_cache_mutex_t *_oidc_metrics_global_mutex = NULL;
// pointer to the thread that periodically writes the locally gathered metrics to shared memory
static apr_thread_t *_oidc_metrics_thread = NULL;
// pid that owns _oidc_metrics_thread; used to tell a post-fork stale handle from a same-process double-init
static pid_t _oidc_metrics_thread_pid = 0;
// local in-memory cached metrics
static oidc_metrics_t _oidc_metrics = {NULL, NULL, NULL};
// mutex to protect the local metrics hash table
static oidc_cache_mutex_t *_oidc_metrics_process_mutex = NULL;

// default shared memory write interval in seconds
#define OIDC_METRICS_CACHE_STORAGE_INTERVAL_DEFAULT 5000

// maximum length of the string representation of the global JSON metrics data in shared memory
//   1024 sample size (compact, long keys, large json_int values, no description), timing + counter
//   256 number of individual metrics collected
//     4 number of vhosts supported
#define OIDC_METRICS_CACHE_JSON_MAX_DEFAULT (1024 * 256 * 4)

// NB: "label" is the millisecond le= label of the legacy (deprecated) Prometheus families,
//     "label_seconds" the seconds le= label of the idiomatic _seconds families
typedef struct oidc_metrics_bucket_t {
  const char *name;
  const char *label;
  const char *label_seconds;
  apr_time_t threshold;
} oidc_metrics_bucket_t;

// clang-format off

static oidc_metrics_bucket_t _oidc_metric_buckets[] = {
  { "le01", "le=\"0.1\"", "le=\"0.0001\"", 100 },
  { "le05", "le=\"0.5\"", "le=\"0.0005\"", 500 },
  { "le1", "le=\"1\"", "le=\"0.001\"", apr_time_from_msec(1) },
  { "le5", "le=\"5\"", "le=\"0.005\"", apr_time_from_msec(5) },
  { "le10", "le=\"10\"", "le=\"0.01\"", apr_time_from_msec(10) },
  { "le50", "le=\"50\"", "le=\"0.05\"", apr_time_from_msec(50) },
  { "le100", "le=\"100\"", "le=\"0.1\"", apr_time_from_msec(100) },
  { "le500", "le=\"500\"", "le=\"0.5\"", apr_time_from_msec(500) },
  { "le1000", "le=\"1000\"", "le=\"1\"", apr_time_from_msec(1000) },
    { "le5000", "le=\"5000\"", "le=\"5\"", apr_time_from_msec(5000) },
    { "le10000", "le=\"10000\"", "le=\"10\"", apr_time_from_msec(10000) },
    { "le30000", "le=\"30000\"", "le=\"30\"", apr_time_from_msec(30000) },
    { "inf", "le=\"+Inf\"", "le=\"+Inf\"", 0 }
};

// clang-format on

#define OIDC_METRICS_BUCKET_NUM ((int)(sizeof(_oidc_metric_buckets) / sizeof(oidc_metrics_bucket_t)))

// NB: matters for Prometheus formatting
#define OIDC_METRICS_SUM "sum"
#define OIDC_METRICS_COUNT "count"

#define OIDC_METRICS_SPECS "specs"

#define OIDC_METRICS_JSON_CLASS_NAME "class"
#define OIDC_METRICS_JSON_METRIC_NAME "name"
#define OIDC_METRICS_JSON_DESC "desc"

#define OIDC_METRICS_TIMINGS "timings"
#define OIDC_METRICS_COUNTERS "counters"

/*
 * convert a Jansson number to a string: apr_psprintf does not understand OIDC_JSON_INT_FORMAT
 * ("%lld"), APR spells a 64-bit integer as APR_INT64_T_FMT, and oidc_json_int_t is at most 64
 * bits wide
 */
static inline char *_json_int2str(apr_pool_t *pool, oidc_json_int_t n) {
  return apr_psprintf(pool, "%" APR_INT64_T_FMT, (apr_int64_t)n);
}

#define OIDC_METRICS_INT_MAX OIDC_JSON_INT_MAX

/*
 * check Jansson specific integer/long number overrun
 */
static inline int _is_overflow(server_rec *s, oidc_json_int_t cur, oidc_json_int_t add) {
  if (add > (OIDC_METRICS_INT_MAX - cur)) {
    oidc_swarn(s,
         "reset metrics since the size (%s) of the integer value would be larger than the "
         "JSON/libjansson maximum "
         "(%s)",
         _json_int2str(s->process->pool, add), _json_int2str(s->process->pool, OIDC_METRICS_INT_MAX));
    return 1;
  }
  return 0;
}

// single counter container
typedef struct oidc_metrics_counter_t {
  oidc_json_int_t count;
} oidc_metrics_counter_t;

// single timing stats container
typedef struct oidc_metrics_timing_t {
  oidc_json_int_t buckets[OIDC_METRICS_BUCKET_NUM];
  apr_time_t sum;
  oidc_json_int_t count;
} oidc_metrics_timing_t;

/*
 * forget the local collection pool when it goes away: it is a child of the process pool, and at exit
 * apr_pool_destroy() of the process pool destroys it before pconf (an older sibling) and so before the pconf
 * cleanup that runs oidc_metrics_cleanup(), which must then not destroy it a second time
 */
static apr_status_t oidc_metrics_local_pool_cleanup(void *data) {
  _oidc_metrics.pool = NULL;
  _oidc_metrics.counters = NULL;
  _oidc_metrics.timings = NULL;
  return APR_SUCCESS;
}

/*
 * Reinitialize the per-process collection tables in a dedicated child pool.
 * Clearing this pool after a flush reclaims all nested hashes, keys and values
 * in one operation; the process mutex must be held once request threads exist.
 */
static apr_byte_t oidc_metrics_local_reset(server_rec *s) {
  apr_pool_t *pool = _oidc_metrics.pool;

  if (pool == NULL) {
    if (apr_pool_create(&pool, s->process->pool) != APR_SUCCESS) {
      oidc_serror(s, "apr_pool_create failed: cannot initialize local metrics");
      return FALSE;
    }
  } else {
    /* NB: this runs (and so consumes) oidc_metrics_local_pool_cleanup as well, hence the re-registration */
    apr_pool_clear(pool);
  }
  apr_pool_cleanup_register(pool, NULL, oidc_metrics_local_pool_cleanup, apr_pool_cleanup_null);

  _oidc_metrics.pool = pool;
  _oidc_metrics.counters = apr_hash_make(pool);
  _oidc_metrics.timings = apr_hash_make(pool);
  return TRUE;
}

// context holder for parsing valid classnames
typedef struct oidc_metrics_add_classname_ctx_t {
  apr_pool_t *pool;
  char **valid_names;
} oidc_metrics_add_classname_ctx_t;

/*
 * loop function for parsing valid classnames
 */
static int oidc_metrics_add_classnames(void *rec, const char *key, const char *value) {
  oidc_metrics_add_classname_ctx_t *ctx = (oidc_metrics_add_classname_ctx_t *)rec;
  *ctx->valid_names = apr_psprintf(ctx->pool, "%s%s%s", *ctx->valid_names ? *ctx->valid_names : "",
           *ctx->valid_names ? " | " : "", value);
  return 1;
}

/*
 * check if the provided value is a valid classname
 */
apr_byte_t oidc_metrics_is_valid_classname(apr_pool_t *pool, const char *name, char **valid_names) {
  int i = 0;
  int n = 0;
  apr_table_t *names = apr_table_make(pool, 1);
  oidc_metrics_add_classname_ctx_t ctx = {pool, valid_names};

  n = sizeof(_oidc_metrics_timings_info) / sizeof(oidc_metrics_timing_info_t);
  for (i = 0; i < n; i++) {
    apr_table_set(names, _oidc_metrics_timings_info[i].class_name,
            _oidc_metrics_timings_info[i].class_name);
  }
  n = sizeof(_oidc_metrics_counters_info) / sizeof(oidc_metrics_counter_info_t);
  for (i = 0; i < n; i++) {
    // NB: "claim" is the only counter class with a name/value option and is special-cased here (rather
    // than driven from a static list)
    if (_oidc_strcmp(_oidc_metrics_counters_info[i].class_name, "claim") != 0)
      apr_table_set(names, _oidc_metrics_counters_info[i].class_name,
              _oidc_metrics_counters_info[i].class_name);
  }

  *valid_names = NULL;
  apr_table_do(oidc_metrics_add_classnames, &ctx, names, NULL);
  *valid_names = apr_psprintf(pool, "%s%s%s", *valid_names ? *valid_names : "", *valid_names ? " | " : "",
            "claim.id_token.* | claim.userinfo.*");

  return apr_table_get(names, name) ? TRUE
            : ((_oidc_strstr(name, "claim.id_token.") != NULL) ||
               (_oidc_strstr(name, "claim.userinfo.") != NULL));
}

/*
 * retrieve environment variable integer with default setting
 */
static inline int _oidc_metrics_get_env_int(const char *name, int dval) {
  return _oidc_str_to_int(getenv(name), dval);
}

#define OIDC_METRICS_CACHE_JSON_MAX_ENV_VAR "OIDC_METRICS_CACHE_JSON_MAX"

/*
 * fixed self-health header kept in the shared memory segment in front of the JSON metrics data;
 * 32-bit members only so the apr_atomic_*32 functions cover every supported APR version
 */
typedef struct oidc_metrics_shm_hdr_t {
  // number of flushes/resets that could not be stored because the JSON data outgrew the segment
  apr_uint32_t flush_errors;
  // seconds-since-epoch timestamp of the last completed flush cycle of this process' flush thread
  apr_uint32_t last_flush;
} oidc_metrics_shm_hdr_t;

#define OIDC_METRICS_SHM_HDR_SIZE (APR_ALIGN_DEFAULT(sizeof(oidc_metrics_shm_hdr_t)))

static inline oidc_metrics_shm_hdr_t *_oidc_metrics_shm_hdr(void) {
  return (oidc_metrics_shm_hdr_t *)apr_shm_baseaddr_get(_oidc_metrics_cache);
}

static inline char *_oidc_metrics_shm_json(void) {
  return ((char *)apr_shm_baseaddr_get(_oidc_metrics_cache)) + OIDC_METRICS_SHM_HDR_SIZE;
}

static apr_size_t _g_oidc_metrics_shm_size = 0;

/*
 * get the maximum size of the serialized JSON metrics data: directive, environment variable or default
 */
static inline apr_size_t _oidc_metrics_shm_size(server_rec *s) {
  if (_g_oidc_metrics_shm_size == 0) {
    const oidc_cfg_t *cfg = ap_get_module_config(s->module_config, &auth_openidc_module);
    int n = oidc_cfg_metrics_cache_json_max_get(cfg);
    if (n == OIDC_CONFIG_POS_INT_UNSET)
      n = _oidc_metrics_get_env_int(OIDC_METRICS_CACHE_JSON_MAX_ENV_VAR,
                  OIDC_METRICS_CACHE_JSON_MAX_DEFAULT);
    if ((n < 1) || (n > 1024 * 256 * 4 * 100)) {
      oidc_serror(s, "environment value %s out of bounds, fallback to default",
            OIDC_METRICS_CACHE_JSON_MAX_ENV_VAR);
      _g_oidc_metrics_shm_size = OIDC_METRICS_CACHE_JSON_MAX_DEFAULT;
    } else {
      _g_oidc_metrics_shm_size = n;
    }
  }
  return _g_oidc_metrics_shm_size;
}

/*
 * retrieve the (JSON) serialized (global) metrics data from shared memory
 */
static inline char *_oidc_metrics_storage_get(server_rec *s, apr_pool_t *pool) {
  const char *p = _oidc_metrics_shm_json();
  return p && (*p != 0) ? apr_pstrndup(pool, p, _oidc_metrics_shm_size(s)) : NULL;
}

/*
 * store the serialized (global) metrics data in shared memory
 */
static inline void _oidc_metrics_storage_set(server_rec *s, const char *value) {
  char *p = _oidc_metrics_shm_json();
  if (value) {
    apr_size_t n = _oidc_strlen(value) + 1;
    if (n > _oidc_metrics_shm_size(s)) {
      apr_atomic_inc32(&_oidc_metrics_shm_hdr()->flush_errors);
      oidc_serror(s,
            "json value too large: set or increase " OIDCMetricsCacheJsonMax
            " (or system environment variable %s) to a value "
            "larger than %" APR_SIZE_T_FMT,
            OIDC_METRICS_CACHE_JSON_MAX_ENV_VAR, _oidc_metrics_shm_size(s));
    } else {
      _oidc_memcpy(p, value, n);
    }
  } else {
    *p = 0;
  }
}

/*
 * parse a string into a JSON object
 */
static oidc_json_t *oidc_metrics_json_load(apr_pool_t *pool, const char *s_json, char **s_err) {
  oidc_json_t *json = NULL;
  if (s_json == NULL)
    s_json = "{}";
  oidc_json_parse(pool, s_json, 0, &json, s_err);
  return json;
}

/*
 * parse a string into a JSON object in a server_rec context
 */
static oidc_json_t *oidc_metrics_json_parse_s(server_rec *s, apr_pool_t *pool, const char *s_json) {
  char *s_err = NULL;
  oidc_json_t *json = oidc_metrics_json_load(pool, s_json, &s_err);
  if (json == NULL)
    oidc_serror(s, "JSON parsing failed: %s", s_err);
  return json;
}

/*
 * recursively reset all integer leaf values within a JSON object tree to 0
 */
static void oidc_metrics_reset_integer_tree(oidc_json_t *json) {
  void *iter = oidc_json_object_iter(json);
  while (iter) {
    oidc_json_t *value = oidc_json_object_iter_value(iter);
    if (oidc_json_is_integer(value))
      oidc_json_integer_set(value, 0);
    else if (oidc_json_is_object(value))
      oidc_metrics_reset_integer_tree(value);
    iter = oidc_json_object_iter_next(json, iter);
  }
}

/*
 * reset a single timings entry (buckets, sum, count) to 0
 */
static void oidc_metrics_reset_timer(oidc_json_t *j_entry) {
  for (int i = 0; i < OIDC_METRICS_BUCKET_NUM; i++)
    oidc_json_object_set_new(j_entry, _oidc_metric_buckets[i].name, oidc_json_integer(0));
  oidc_json_object_set_new(j_entry, OIDC_METRICS_SUM, oidc_json_integer(0));
  oidc_json_object_set_new(j_entry, OIDC_METRICS_COUNT, oidc_json_integer(0));
}

/*
 * reset all counter and timer entries for a single server in the JSON data
 */
static void oidc_metrics_reset_server(const oidc_json_t *j_server) {
  oidc_json_t *j_entries = NULL;
  void *iter = NULL;

  j_entries = oidc_json_object_get(j_server, OIDC_METRICS_COUNTERS);
  if (j_entries)
    oidc_metrics_reset_integer_tree(j_entries);

  j_entries = oidc_json_object_get(j_server, OIDC_METRICS_TIMINGS);
  iter = oidc_json_object_iter(j_entries);
  while (iter) {
    oidc_metrics_reset_timer(oidc_json_object_iter_value(iter));
    iter = oidc_json_object_iter_next(j_entries, iter);
  }
}

/*
 * reset the serialized (global) metrics data in shared memory
 */
static inline void oidc_metrics_storage_reset(server_rec *s, apr_pool_t *pool) {
  const char *s_json = NULL;
  oidc_json_t *json = NULL;
  void *iter = NULL;

  /* get the global stringified JSON metrics */
  s_json = _oidc_metrics_storage_get(s, pool);

  /* parse the metrics string to JSON */
  json = oidc_metrics_json_parse_s(s, pool, s_json);
  if (json == NULL)
    json = oidc_json_object();

  iter = oidc_json_object_iter(json);
  while (iter) {
    oidc_metrics_reset_server(oidc_json_object_iter_value(iter));
    iter = oidc_json_object_iter_next(json, iter);
  }

  /* serialize the metrics data, preserve order is required for Prometheus */
  s_json = oidc_json_encode(pool, json, OIDC_JSON_COMPACT | OIDC_JSON_PRESERVE_ORDER);

  /* free the JSON data */
  oidc_json_decref(json);

  /* store the serialized metrics data in shared memory */
  _oidc_metrics_storage_set(s, s_json);
}

/*
 * create a new timings entry in the collected JSON data
 */
static oidc_json_t *oidc_metrics_timings_new(const oidc_metrics_timing_t *timing) {
  oidc_json_t *entry = oidc_json_object();
  for (int i = 0; i < OIDC_METRICS_BUCKET_NUM; i++)
    oidc_json_object_set_new(entry, _oidc_metric_buckets[i].name, oidc_json_integer(timing->buckets[i]));
  /* the sum is kept in microseconds so the _seconds Prometheus family is exact; the
   * (documented) json format converts to milliseconds at presentation time */
  oidc_json_object_set_new(entry, OIDC_METRICS_SUM, oidc_json_integer(timing->sum));
  oidc_json_object_set_new(entry, OIDC_METRICS_COUNT, oidc_json_integer(timing->count));
  return entry;
}

/*
 * update an entry in the collected JSON data
 */
static void oidc_metrics_timings_update(server_rec *s, const oidc_json_t *entry, const oidc_metrics_timing_t *timing) {
  oidc_json_t *j_member = NULL;
  oidc_json_int_t n = 0;
  oidc_json_int_t v = 0;

  for (int i = 0; i < OIDC_METRICS_BUCKET_NUM; i++) {
    j_member = oidc_json_object_get(entry, _oidc_metric_buckets[i].name);
    oidc_json_integer_set(j_member, oidc_json_integer_value(j_member) + timing->buckets[i]);
  }

  j_member = oidc_json_object_get(entry, OIDC_METRICS_SUM);
  n = oidc_json_integer_value(j_member);

  /* microseconds, matching oidc_metrics_timings_new */
  v = timing->sum;
  if (_is_overflow(s, n, v))
    n = 0;

  oidc_json_integer_set(j_member, n + v);

  j_member = oidc_json_object_get(entry, OIDC_METRICS_COUNT);
  n = oidc_json_integer_value(j_member);
  oidc_json_integer_set(j_member, n + timing->count);
}

#define OIDC_METRICS_VALUE_DEFAULT "_"

/*
 * value helper to make sure it is not empty
 */
static inline const char *_metrics_value2key(const char *value) {
  return (value && _oidc_strcmp(value, "") != 0) ? value : OIDC_METRICS_VALUE_DEFAULT;
}

/*
 * create a new counter entry in the collected JSON data
 */
static oidc_json_t *oidc_metrics_counter_new(apr_hash_t *htable) {
  oidc_metrics_counter_t *counter = NULL;
  char *value = NULL;
  oidc_json_t *j_values = NULL;
  for (apr_hash_index_t *hi = apr_hash_first(NULL, htable); hi; hi = apr_hash_next(hi)) {
    apr_hash_this(hi, (const void **)&value, NULL, (void **)&counter);
    if (_oidc_strcmp(value, OIDC_METRICS_VALUE_DEFAULT) == 0) {
      j_values = oidc_json_integer(counter->count);
    } else {
      if (j_values == NULL)
        j_values = oidc_json_object();
      oidc_json_object_set_new(j_values, value, oidc_json_integer(counter->count));
    }
  }
  return j_values;
}

/*
 * update a counter entry in the collected JSON data
 */
static void oidc_metrics_counter_update(server_rec *s, oidc_json_t *j_counter, apr_hash_t *htable) {
  oidc_json_int_t v = 0;
  oidc_metrics_counter_t *counter = NULL;
  char *value = NULL;
  oidc_json_t *j_value = NULL;
  for (apr_hash_index_t *hi = apr_hash_first(NULL, htable); hi; hi = apr_hash_next(hi)) {
    apr_hash_this(hi, (const void **)&value, NULL, (void **)&counter);
    if (_oidc_strcmp(value, OIDC_METRICS_VALUE_DEFAULT) == 0) {
      j_value = j_counter;
    } else {
      j_value = oidc_json_object_get(j_counter, value);
      if (j_value == NULL) {
        oidc_json_object_set_new(j_counter, value, oidc_json_integer(counter->count));
        continue;
      }
    }
    v = oidc_json_integer_value(j_value);
    if (_is_overflow(s, v, counter->count))
      v = 0;
    oidc_json_integer_set(j_value, v + counter->count);
  }
}

/*
 * get or create the vhost entry in the global metrics
 */
static oidc_json_t *oidc_metrics_server_get(oidc_json_t *json, const char *name) {
  oidc_json_t *j_server = oidc_json_object_get(json, name);
  if (j_server == NULL) {
    j_server = oidc_json_object();
    oidc_json_object_set_new(j_server, OIDC_METRICS_COUNTERS, oidc_json_object());
    oidc_json_object_set_new(j_server, OIDC_METRICS_TIMINGS, oidc_json_object());
    oidc_json_object_set_new(json, name, j_server);
  }
  return j_server;
}

/*
 * convert an enum type value to its corresponding string
 */
static inline char *_oidc_metrics_type_name2key(apr_pool_t *pool, unsigned int type, const char *name) {
  return (name == NULL) ? apr_psprintf(pool, "%u", type) : apr_psprintf(pool, "%u.%s", type, name);
}

/*
 * look up the info table entry for the enum type in a "<type>[.<name>]" key: the keys come out of the
 * JSON in the shared memory segment that this module writes itself, but a corrupted or stale segment
 * (or a key that does not parse) must not turn into an out-of-bounds index into the info tables, so
 * the type is checked against the table size right where it indexes it; NULL for a key that does not
 * name a type this build knows, which the callers skip
 */
static inline const oidc_metrics_counter_info_t *_oidc_metrics_key2counter_info(const char *key) {
  unsigned int type = 0;
  if ((key == NULL) || (sscanf(key, "%u", &type) != 1) || (type >= OM_NUMBER_OF_COUNTERS))
    return NULL;
  return &_oidc_metrics_counters_info[type];
}

static inline const oidc_metrics_timing_info_t *_oidc_metrics_key2timing_info(const char *key) {
  unsigned int type = 0;
  if ((key == NULL) || (sscanf(key, "%u", &type) != 1) || (type >= OM_NUMBER_OF_TIMINGS))
    return NULL;
  return &_oidc_metrics_timings_info[type];
}

/*
 * update an existing counter under parent[key] or create a new one from counter_hash
 */
static void oidc_metrics_counter_set_or_update(server_rec *s, oidc_json_t *parent, const char *key,
                 apr_hash_t *counter_hash) {
  oidc_json_t *j_counter = oidc_json_object_get(parent, key);
  if (j_counter != NULL)
    oidc_metrics_counter_update(s, j_counter, counter_hash);
  else
    oidc_json_object_set_new(parent, key, oidc_metrics_counter_new(counter_hash));
}

/*
 * merge a single local counter entry into the global counters object;
 * keys of the form "class.name" are nested as j_counters[class][name]
 */
static void oidc_metrics_store_counter_entry(server_rec *s, apr_pool_t *pool, oidc_json_t *j_counters, const char *key,
               apr_hash_t *counter_hash) {
  const char *class_name = apr_pstrdup(pool, key);
  char *name = _oidc_strstr(class_name, ".");
  oidc_json_t *j_names = NULL;

  if (name == NULL) {
    oidc_metrics_counter_set_or_update(s, j_counters, class_name, counter_hash);
    return;
  }

  *name++ = '\0';
  j_names = oidc_json_object_get(j_counters, class_name);
  if (j_names == NULL) {
    j_names = oidc_json_object();
    oidc_json_object_set_new(j_names, name, oidc_metrics_counter_new(counter_hash));
    oidc_json_object_set_new(j_counters, class_name, j_names);
    return;
  }

  oidc_metrics_counter_set_or_update(s, j_names, name, counter_hash);
}

/*
 * merge all locally collected counters into the global JSON
 */
static void oidc_metrics_store_counters(server_rec *s, apr_pool_t *pool, oidc_json_t *json) {
  const char *name = NULL;
  const char *key = NULL;
  apr_hash_t *server_hash = NULL;
  apr_hash_t *counter_hash = NULL;
  oidc_json_t *j_counters = NULL;

  for (apr_hash_index_t *hi1 = apr_hash_first(NULL, _oidc_metrics.counters); hi1; hi1 = apr_hash_next(hi1)) {
    apr_hash_this(hi1, (const void **)&name, NULL, (void **)&server_hash);
    j_counters = oidc_json_object_get(oidc_metrics_server_get(json, name), OIDC_METRICS_COUNTERS);
    for (apr_hash_index_t *hi2 = apr_hash_first(NULL, server_hash); hi2; hi2 = apr_hash_next(hi2)) {
      apr_hash_this(hi2, (const void **)&key, NULL, (void **)&counter_hash);
      oidc_metrics_store_counter_entry(s, pool, j_counters, key, counter_hash);
    }
  }
}

/*
 * merge all locally collected timings into the global JSON
 */
static void oidc_metrics_store_timings(server_rec *s, oidc_json_t *json) {
  const char *name = NULL;
  const char *key = NULL;
  apr_hash_t *server_hash = NULL;
  oidc_metrics_timing_t *timing = NULL;
  oidc_json_t *j_timings = NULL;
  const oidc_json_t *j_timer = NULL;

  for (apr_hash_index_t *hi1 = apr_hash_first(NULL, _oidc_metrics.timings); hi1; hi1 = apr_hash_next(hi1)) {
    apr_hash_this(hi1, (const void **)&name, NULL, (void **)&server_hash);
    j_timings = oidc_json_object_get(oidc_metrics_server_get(json, name), OIDC_METRICS_TIMINGS);
    for (apr_hash_index_t *hi2 = apr_hash_first(NULL, server_hash); hi2; hi2 = apr_hash_next(hi2)) {
      apr_hash_this(hi2, (const void **)&key, NULL, (void **)&timing);
      j_timer = oidc_json_object_get(j_timings, key);
      if (j_timer != NULL)
        oidc_metrics_timings_update(s, j_timer, timing);
      else
        oidc_json_object_set_new(j_timings, key, oidc_metrics_timings_new(timing));
    }
  }
}

/*
 * flush the locally gathered metrics data into the global data kept in shared memory
 */
static apr_byte_t oidc_metrics_store(server_rec *s) {
  const char *s_json = NULL;
  oidc_json_t *json = NULL;
  apr_pool_t *pool = NULL;

  if ((apr_hash_count(_oidc_metrics.counters) == 0) && (apr_hash_count(_oidc_metrics.timings) == 0))
    return TRUE;

  /* everything below is scratch: the whole document is read, re-serialized and copied
   * into shared memory on every flush, so it must not come from the process pool,
   * which is never cleared for the lifetime of the server */
  if (apr_pool_create(&pool, s->process->pool) != APR_SUCCESS) {
    oidc_serror(s, "apr_pool_create failed: cannot flush metrics");
    return FALSE;
  }

  /* lock the shared memory for other processes */
  if (oidc_cache_mutex_lock(pool, s, _oidc_metrics_global_mutex) == FALSE) {
    apr_pool_destroy(pool);
    return FALSE;
  }

  /* get the global stringified JSON metrics */
  s_json = _oidc_metrics_storage_get(s, pool);

  /* parse the metrics string to JSON */
  json = oidc_metrics_json_parse_s(s, pool, s_json);
  if (json == NULL)
    json = oidc_json_object();

  oidc_metrics_store_counters(s, pool, json);
  oidc_metrics_store_timings(s, json);

  /* serialize the metrics data, preserve order is required for Prometheus */
  s_json = oidc_json_encode(pool, json, OIDC_JSON_COMPACT | OIDC_JSON_PRESERVE_ORDER);

  /* free the JSON data */
  oidc_json_decref(json);

  /* store the serialized metrics data in shared memory */
  _oidc_metrics_storage_set(s, s_json);

  /* unlock the shared memory for other processes; the transfer already completed, so an unlock
   * failure (logged by the mutex layer) still reports the data as consumed and a later cycle
   * cannot double-count it */
  (void)oidc_cache_mutex_unlock(pool, s, _oidc_metrics_global_mutex);

  apr_pool_destroy(pool);
  return TRUE;
}

#define OIDC_METRICS_CACHE_STORAGE_INTERVAL_ENV_VAR "OIDC_METRICS_CACHE_STORAGE_INTERVAL"

/*
 * obtain the metrics flush interval: directive, environment variable or default
 */
static inline apr_interval_time_t _oidc_metrics_interval(server_rec *s) {
  const oidc_cfg_t *cfg = ap_get_module_config(s->module_config, &auth_openidc_module);
  int n = oidc_cfg_metrics_cache_storage_interval_get(cfg);
  if (n == OIDC_CONFIG_POS_INT_UNSET)
    n = _oidc_metrics_get_env_int(OIDC_METRICS_CACHE_STORAGE_INTERVAL_ENV_VAR,
                OIDC_METRICS_CACHE_STORAGE_INTERVAL_DEFAULT);
  return apr_time_from_msec(n);
}

#define OIDC_METRICS_POLL_INTERVAL 250

/*
 * thread that periodically writes the local data into the shared memory
 */
static void *APR_THREAD_FUNC oidc_metrics_thread_run(apr_thread_t *thread, void *data) {
  server_rec *s = (server_rec *)data;

  /* sleep for a short random time <0.1s so child processes write-lock on a different frequency */
  apr_sleep(apr_time_from_msec(oidc_util_rand_int(100)));

  /* split the flush interval into POLL_INTERVAL-sized ticks so shutdown is observed quickly; if the
   * configured interval is shorter than POLL_INTERVAL, use it as the tick directly so we still flush
   * on schedule rather than busy-looping with n=0 */
  apr_interval_time_t interval = _oidc_metrics_interval(s);
  /* a misconfigured env var parses to 0 (or negative) — fall back to the default to avoid 0/0 below */
  if (interval <= 0)
    interval = apr_time_from_msec(OIDC_METRICS_CACHE_STORAGE_INTERVAL_DEFAULT);
  apr_interval_time_t tick = apr_time_from_msec(OIDC_METRICS_POLL_INTERVAL);
  if (tick > interval)
    tick = interval;
  int n = (int)(interval / tick);

  /* NB: test for exit only after flushing, because we need to write our local metrics into the cache before
   * exiting, also when asked to exit during the random sleep above */
  do {

    /* break up the sleep interval in short intervals so we can exit timely fashion without confusing Apache
     * at shutdown */
    for (int i = 0; (i < n) && (apr_atomic_read32(&_oidc_metrics_thread_exit) == 0); i++)
      apr_sleep(tick);

    /* lock the mutex that protects the locally cached metrics */
    if (oidc_cache_mutex_lock(s->process->pool, s, _oidc_metrics_process_mutex) == FALSE)
      continue;

    /* flush the locally cached metrics into the global shared memory */
    if (oidc_metrics_store(s) == TRUE) {
      /* record that a flush cycle completed, for the metrics self-health output */
      apr_atomic_set32(&_oidc_metrics_shm_hdr()->last_flush,
           (apr_uint32_t)apr_time_sec(apr_time_now()));

      /* release all allocations made while gathering this flush interval */
      oidc_metrics_local_reset(s);
    }

    /* unlock the mutex that protects the locally cached metrics */
    if (oidc_cache_mutex_unlock(s->process->pool, s, _oidc_metrics_process_mutex) == FALSE)
      break;

  } while (apr_atomic_read32(&_oidc_metrics_thread_exit) == 0);

  /* NB: don't call apr_thread_exit here because it seems that Apache is cleaning up its own threads */

  return NULL;
}

/*
 * stop the flush thread, which writes what it gathered into shared memory one last time on its way out; a handle
 * inherited over fork() names a thread that does not exist in this process and is just dropped
 */
static void oidc_metrics_thread_stop(server_rec *s) {
  apr_status_t rv = APR_SUCCESS;

  if ((_oidc_metrics_thread != NULL) && (_oidc_metrics_thread_pid == getpid())) {
    apr_atomic_set32(&_oidc_metrics_thread_exit, 1);
    if (apr_thread_join(&rv, _oidc_metrics_thread) != APR_SUCCESS)
      oidc_serror(s, "apr_thread_join failed");
    apr_atomic_set32(&_oidc_metrics_thread_exit, 0);
  }
  _oidc_metrics_thread = NULL;
  _oidc_metrics_thread_pid = 0;
}

/*
 * child pool cleanup: stop the flush thread before the child exits; leaving that to oidc_metrics_cleanup() is too
 * late, since that is a pconf cleanup which at exit only runs after apr_terminate() has already destroyed the
 * local collection pool that the thread clears after every flush (see oidc_metrics_local_pool_cleanup), and a
 * flush landing in between crashes in apr_pool_clear()
 */
static apr_status_t oidc_metrics_child_cleanup(void *data) {
  oidc_metrics_thread_stop((server_rec *)data);
  return APR_SUCCESS;
}

/*
 * server config handlers
 */

/*
 * NB: global, yet called for each vhost that has metrics enabled!
 */
apr_byte_t oidc_metrics_post_config(apr_pool_t *pool, server_rec *s) {

  /* make sure it gets executed exactly once! */
  if (_oidc_metrics_cache != NULL)
    return TRUE;

  /* create the shared memory segment that holds the self-health header plus the
   * stringified JSON formatted metrics data */
  if (apr_shm_create(&_oidc_metrics_cache, OIDC_METRICS_SHM_HDR_SIZE + _oidc_metrics_shm_size(s), NULL,
         s->process->pool) != APR_SUCCESS)
    return FALSE;
  if (_oidc_metrics_cache == NULL)
    return FALSE;

  /* initialize the shared memory segment to 0 */
  char *p = apr_shm_baseaddr_get(_oidc_metrics_cache);
  _oidc_memset(p, 0, OIDC_METRICS_SHM_HDR_SIZE + _oidc_metrics_shm_size(s));

  /* flag this as the parent, for shared memory cleanup purposes and "multiple child-init calls" detection */
  _oidc_metrics_is_parent = TRUE;

  /* create the pool and hashtables that hold local metrics data */
  if (oidc_metrics_local_reset(s) == FALSE)
    return FALSE;

  /* create and initialize the mutex that guards the shared-memory data */
  _oidc_metrics_global_mutex = oidc_cache_mutex_create(s->process->pool, TRUE);
  if (_oidc_metrics_global_mutex == NULL)
    return FALSE;
  if (oidc_cache_mutex_post_config(s->process->pool, s, _oidc_metrics_global_mutex, "metrics-global") == FALSE)
    return FALSE;

  /* create and initialize the mutex that guards the local metrics tables */
  _oidc_metrics_process_mutex = oidc_cache_mutex_create(s->process->pool, FALSE);
  if (_oidc_metrics_process_mutex == NULL)
    return FALSE;
  if (oidc_cache_mutex_post_config(s->process->pool, s, _oidc_metrics_process_mutex, "metrics-process") == FALSE)
    return FALSE;

  /* NB: no flush thread in the parent: samples are only ever recorded while handling a request, which the parent
   * does not do; the processes that do (including a -X single process) start one in oidc_metrics_child_init, and
   * a parent thread would race apr_pool_destroy() of the process pool at shutdown, which happens before the pconf
   * cleanup that would stop it */

  return TRUE;
}

/*
 * NB: global, yet called for each vhost that has metrics enabled!
 */
apr_status_t oidc_metrics_child_init(apr_pool_t *p, server_rec *s) {

  /* make sure this executes only once per child */
  if (_oidc_metrics_is_parent == FALSE)
    return APR_SUCCESS;

  if (oidc_cache_mutex_child_init(p, s, _oidc_metrics_global_mutex) != APR_SUCCESS)
    return APR_EGENERAL;

  if (oidc_cache_mutex_child_init(p, s, _oidc_metrics_process_mutex) != APR_SUCCESS)
    return APR_EGENERAL;

  /* stop an existing same-process flush thread; discard inherited handles after a fork */
  oidc_metrics_thread_stop(s);
  if (oidc_metrics_local_reset(s) == FALSE)
    return APR_EGENERAL;

  /* create the thread only after all state that it accesses has been initialized */
  if (apr_thread_create(&_oidc_metrics_thread, NULL, oidc_metrics_thread_run, s, s->process->pool) != APR_SUCCESS)
    return APR_EGENERAL;
  _oidc_metrics_thread_pid = getpid();

  /* stop it again when the child pool goes; registered after the mutex child_init calls above so that, cleanups
   * running last-in-first-out, the thread's final flush still finds them initialized */
  apr_pool_cleanup_register(p, s, oidc_metrics_child_cleanup, apr_pool_cleanup_null);

  /* flag this is a child */
  _oidc_metrics_is_parent = FALSE;

  return APR_SUCCESS;
}

/*
 * NB: global, yet called for each vhost that has metrics enabled!
 */
apr_status_t oidc_metrics_cleanup(server_rec *s) {

  /* make sure it gets executed exactly once! NB: not keyed on the flush thread, which the parent does not run
   * and a child has already stopped in oidc_metrics_child_cleanup by the time it gets here */
  if (_oidc_metrics_cache == NULL)
    return APR_SUCCESS;

  /* stop the collector thread, if it still runs */
  oidc_metrics_thread_stop(s);

  /* delete the shared memory segment if we are in the parent process */
  if (_oidc_metrics_is_parent == TRUE)
    apr_shm_destroy(_oidc_metrics_cache);
  _oidc_metrics_cache = NULL;
  /* the cached segment size lives exactly as long as the segment: the next post_config (a graceful
   * restart, or the next unit test in the same process) sizes a new one from its own configuration */
  _g_oidc_metrics_shm_size = 0;

  /* delete the process mutex that guards the local metrics data */
  if (oidc_cache_mutex_destroy(s, _oidc_metrics_process_mutex) == FALSE)
    return APR_EGENERAL;
  _oidc_metrics_process_mutex = NULL;

  /* delete the process mutex that guards the global shared memory segment */
  if (oidc_cache_mutex_destroy(s, _oidc_metrics_global_mutex) == FALSE)
    return APR_EGENERAL;
  _oidc_metrics_global_mutex = NULL;

  if (_oidc_metrics.pool != NULL)
    apr_pool_destroy(_oidc_metrics.pool);
  _oidc_metrics.pool = NULL;
  _oidc_metrics.counters = NULL;
  _oidc_metrics.timings = NULL;

  return APR_SUCCESS;
}

/*
 * sampling
 */

/*
 * obtain the local metrics hashtable for the current vhost
 */
static inline apr_hash_t *_oidc_metrics_server_hash(const request_rec *r, apr_hash_t *table) {
  apr_hash_t *server_hash = NULL;
  const char *name = "_default_";

  /* obtain the server name */
  if (r->server->server_hostname)
    name = r->server->server_hostname;

  /* get the entry to the vhost record, or newly create it */
  server_hash = apr_hash_get(table, name, APR_HASH_KEY_STRING);
  if (server_hash == NULL) {
    server_hash = apr_hash_make(_oidc_metrics.pool);
    apr_hash_set(table, name, APR_HASH_KEY_STRING, server_hash);
  }

  return server_hash;
}

/*
 * retrieve or create a local timing for the specified type
 */
static inline oidc_metrics_timing_t *_oidc_metrics_timing_get(request_rec *r, unsigned int type) {
  oidc_metrics_timing_t *result = NULL;
  /* NB: the lookup key is request-scoped; only the copy the hash retains on insert
   * below is allocated process-wide, since that pool is never cleared */
  const char *key = _oidc_metrics_type_name2key(r->pool, type, NULL);
  apr_hash_t *server_hash = _oidc_metrics_server_hash(r, _oidc_metrics.timings);
  /* get the entry to the specified metric */
  result = apr_hash_get(server_hash, key, APR_HASH_KEY_STRING);
  if (result == NULL) {
    result = apr_pcalloc(_oidc_metrics.pool, sizeof(oidc_metrics_timing_t));
    apr_hash_set(server_hash, apr_pstrdup(_oidc_metrics.pool, key), APR_HASH_KEY_STRING, result);
  }
  return result;
}

/*
 * retrieve or create a counter from a hashtable of values
 */
static inline oidc_metrics_counter_t *_oidc_metrics_counter_value_get(apr_hash_t *table, const char *value) {
  /* get the entry to the specified metric */
  oidc_metrics_counter_t *result = apr_hash_get(table, value, APR_HASH_KEY_STRING);
  if (result == NULL) {
    result = apr_pcalloc(_oidc_metrics.pool, sizeof(oidc_metrics_counter_t));
    apr_hash_set(table, apr_pstrdup(_oidc_metrics.pool, value), APR_HASH_KEY_STRING, result);
  }
  return result;
}

/*
 * retrieve or create a local counter for the specified type and name
 */
static inline apr_hash_t *_oidc_metrics_counter_get(request_rec *r, unsigned int type, const char *name) {
  apr_hash_t *result = NULL;
  /* NB: request-scoped lookup key, process-wide copy only on insert - see above */
  const char *key = _oidc_metrics_type_name2key(r->pool, type, name);
  apr_hash_t *server_hash = _oidc_metrics_server_hash(r, _oidc_metrics.counters);

  /* get the entry to the specified metric */
  result = apr_hash_get(server_hash, key, APR_HASH_KEY_STRING);
  if (result == NULL) {
    result = apr_hash_make(_oidc_metrics.pool);
    apr_hash_set(server_hash, apr_pstrdup(_oidc_metrics.pool, key), APR_HASH_KEY_STRING, result);
  }

  return result;
}

/*
 * add/increase a counter metric in the locally cached data
 */
void oidc_metrics_counter_inc(request_rec *r, oidc_metrics_counter_type_t type, const char *name, const char *value) {
  oidc_metrics_counter_t *counter = NULL;

  /* lock the local metrics cache hashtable */
  if (oidc_cache_mutex_lock(r->pool, r->server, _oidc_metrics_process_mutex) == FALSE)
    return;

  /* obtain or create the entry for the specified key */
  counter = _oidc_metrics_counter_value_get(_oidc_metrics_counter_get(r, type, name), _metrics_value2key(value));

  /* performance */
  if (counter->count <= 0) {
    // new counter was created just now or reset earlier
    counter->count = 1;
  } else {
    // increase after checking possible overflow
    if (_is_overflow(r->server, counter->count, 1))
      counter->count = 0;
    counter->count++;
  }

  /* unlock the local metrics cache hashtable */
  (void)oidc_cache_mutex_unlock(r->pool, r->server, _oidc_metrics_process_mutex);
}

/*
 * zero out a timings entry (count, sum and all buckets)
 */
static inline void _oidc_metrics_timing_clear(oidc_metrics_timing_t *timing) {
  timing->count = 0;
  timing->sum = 0;
  for (int i = 0; i < OIDC_METRICS_BUCKET_NUM; i++)
    timing->buckets[i] = 0;
}

/*
 * increment every bucket whose threshold covers elapsed (buckets are ordered;
 * threshold == 0 marks the open-ended last bucket)
 */
static inline void _oidc_metrics_timing_buckets_inc(oidc_metrics_timing_t *timing, apr_time_t elapsed) {
  for (int i = 0; i < OIDC_METRICS_BUCKET_NUM; i++) {
    if ((elapsed <= _oidc_metric_buckets[i].threshold) || (_oidc_metric_buckets[i].threshold == 0)) {
      for (int j = i; j < OIDC_METRICS_BUCKET_NUM; j++)
        timing->buckets[j]++;
      break;
    }
  }
}

/*
 * add a metrics timing sample to the locally cached data
 */
void oidc_metrics_timing_add(request_rec *r, oidc_metrics_timing_type_t type, apr_time_t elapsed) {
  oidc_metrics_timing_t *timing = NULL;

  if (elapsed < 0) {
    oidc_warn(r, "discarding metrics timing [%s.%s]: elapsed (%" APR_TIME_T_FMT ") < 0",
        _oidc_metrics_timings_info[type].class_name, _oidc_metrics_timings_info[type].metric_name,
        elapsed);
    return;
  }

  /* lock the local metrics cache hashtable */
  if (oidc_cache_mutex_lock(r->pool, r->server, _oidc_metrics_process_mutex) == FALSE)
    return;

  /* obtain or create the entry for the specified key */
  timing = _oidc_metrics_timing_get(r, type);

  /* reset on overflow; a freshly-created entry already has count/sum/buckets at zero */
  if ((timing->count > 0) && _is_overflow(r->server, timing->sum, elapsed))
    _oidc_metrics_timing_clear(timing);

  _oidc_metrics_timing_buckets_inc(timing, elapsed);
  timing->sum += elapsed;
  timing->count++;

  /* unlock the local metrics cache hashtable */
  (void)oidc_cache_mutex_unlock(r->pool, r->server, _oidc_metrics_process_mutex);
}

/*
 * representation handlers
 */

/*
 * the "class.metric" name of a counter, from its info table entry
 */
static inline char *_oidc_metrics_counter_info2s(apr_pool_t *pool, const oidc_metrics_counter_info_t *info) {
  return apr_psprintf(pool, "%s.%s", info->class_name, info->metric_name);
}

/*
 * the "class.metric" name of a timing, from its info table entry
 */
static inline char *_oidc_metrics_timing_info2s(apr_pool_t *pool, const oidc_metrics_timing_info_t *info) {
  return apr_psprintf(pool, "%s.%s", info->class_name, info->metric_name);
}

/*
 * parse a string into a JSON object in the request_rec context
 */
static oidc_json_t *oidc_metrics_json_parse_r(request_rec *r, const char *s_json) {
  char *s_err = NULL;
  oidc_json_t *json = oidc_metrics_json_load(r->pool, s_json, &s_err);
  if (json == NULL)
    oidc_error(r, "JSON parsing failed: %s", s_err);
  return json;
}

/*
 * JSON with extended descriptions/names
 */
static int oidc_metrics_handle_json(request_rec *r, const char *s_json) {

  oidc_json_t *json = NULL;
  const oidc_json_t *j_server = NULL;
  oidc_json_t *j_timings = NULL;
  oidc_json_t *j_counters = NULL;
  const oidc_json_t *j_timing = NULL;
  oidc_json_t *j_counter = NULL;
  oidc_json_t *o_json = NULL;
  oidc_json_t *o_server = NULL;
  oidc_json_t *o_counters = NULL;
  oidc_json_t *o_counter = NULL;
  oidc_json_t *o_timings = NULL;
  oidc_json_t *o_timing = NULL;
  const char *s_server = NULL;
  const oidc_metrics_counter_info_t *c_info = NULL;
  const oidc_metrics_timing_info_t *t_info = NULL;
  void *i1 = NULL;
  void *i2 = NULL;

  /* parse the metrics string to JSON */
  json = oidc_metrics_json_parse_r(r, s_json);
  if (json == NULL)
    goto end;

  o_json = oidc_json_object();

  i1 = oidc_json_object_iter(json);
  while (i1) {
    s_server = oidc_json_object_iter_key(i1);
    j_server = oidc_json_object_iter_value(i1);

    o_server = oidc_json_object();
    oidc_json_object_set_new(o_json, s_server, o_server);

    j_counters = oidc_json_object_get(j_server, OIDC_METRICS_COUNTERS);
    o_counters = oidc_json_object();
    oidc_json_object_set_new(o_server, OIDC_METRICS_COUNTERS, o_counters);

    i2 = oidc_json_object_iter(j_counters);
    while (i2) {
      c_info = _oidc_metrics_key2counter_info(oidc_json_object_iter_key(i2));
      if (c_info == NULL) {
        /* not a counter this build knows: skip it rather than index past the info table */
        i2 = oidc_json_object_iter_next(j_counters, i2);
        continue;
      }
      j_counter = oidc_json_object_iter_value(i2);
      o_counter = oidc_json_object();
      if (oidc_json_is_integer(j_counter))
        oidc_json_object_set(o_counter, "count", j_counter);
      else
        oidc_json_object_set_new(o_counter, "values", oidc_json_deep_copy(j_counter));
      oidc_json_object_set_new(o_counter, OIDC_METRICS_JSON_CLASS_NAME,
             oidc_json_string(c_info->class_name));
      oidc_json_object_set_new(o_counter, OIDC_METRICS_JSON_METRIC_NAME,
             oidc_json_string(c_info->metric_name));
      oidc_json_object_set_new(o_counter, OIDC_METRICS_JSON_DESC, oidc_json_string(c_info->desc));
      oidc_json_object_set_new(o_counters, _oidc_metrics_counter_info2s(r->pool, c_info), o_counter);
      i2 = oidc_json_object_iter_next(j_counters, i2);
    }

    j_timings = oidc_json_object_get(j_server, OIDC_METRICS_TIMINGS);
    o_timings = oidc_json_object();
    oidc_json_object_set_new(o_server, OIDC_METRICS_TIMINGS, o_timings);

    i2 = oidc_json_object_iter(j_timings);
    while (i2) {
      t_info = _oidc_metrics_key2timing_info(oidc_json_object_iter_key(i2));
      if (t_info == NULL) {
        i2 = oidc_json_object_iter_next(j_timings, i2);
        continue;
      }
      j_timing = oidc_json_object_iter_value(i2);

      o_timing = oidc_json_deep_copy(j_timing);
      /* the shm sum is kept in microseconds; this format documents milliseconds */
      oidc_json_object_set_new(
          o_timing, OIDC_METRICS_SUM,
          oidc_json_integer(
        oidc_json_integer_value(oidc_json_object_get(j_timing, OIDC_METRICS_SUM)) / 1000));
      oidc_json_object_set_new(o_timing, OIDC_METRICS_JSON_CLASS_NAME,
             oidc_json_string(t_info->class_name));
      oidc_json_object_set_new(o_timing, OIDC_METRICS_JSON_METRIC_NAME,
             oidc_json_string(t_info->metric_name));
      oidc_json_object_set_new(o_timing, OIDC_METRICS_JSON_DESC, oidc_json_string(t_info->desc));

      oidc_json_object_set_new(o_timings, _oidc_metrics_timing_info2s(r->pool, t_info), o_timing);

      i2 = oidc_json_object_iter_next(j_timings, i2);
    }
    i1 = oidc_json_object_iter_next(json, i1);
  }

  s_json = oidc_json_encode(r->pool, o_json, OIDC_JSON_COMPACT | OIDC_JSON_PRESERVE_ORDER);

  oidc_json_decref(o_json);
  oidc_json_decref(json);

end:

  /* return the data to the caller */
  return oidc_util_http_send(r, s_json, _oidc_strlen(s_json), OIDC_HTTP_CONTENT_TYPE_JSON, OK);
}

/*
 * dump the internal shared memory segment
 */
static int oidc_metrics_handle_internal(request_rec *r, const char *s_json) {
  if (s_json == NULL)
    return HTTP_NOT_FOUND;
  return oidc_util_http_send(r, s_json, _oidc_strlen(s_json), OIDC_HTTP_CONTENT_TYPE_JSON, OK);
}

#define OIDC_METRICS_SERVER_PARAM "server_name"
#define OIDC_METRICS_COUNTER_PARAM "counter"
#define OIDC_METRICS_NAME_PARAM "name"
#define OIDC_METRICS_VALUE_PARAM "value"

/*
 * find the counter entry in j_counters whose type matches "class.metric" (s_metric_param)
 */
static oidc_json_t *oidc_metrics_status_find_counter(request_rec *r, oidc_json_t *j_counters,
                 const char *s_metric_param) {
  const oidc_metrics_counter_info_t *info = NULL;
  void *iter = oidc_json_object_iter(j_counters);
  while (iter) {
    info = _oidc_metrics_key2counter_info(oidc_json_object_iter_key(iter));
    if ((info != NULL) && (_oidc_strcmp(_oidc_metrics_counter_info2s(r->pool, info), s_metric_param) == 0))
      return oidc_json_object_iter_value(iter);
    iter = oidc_json_object_iter_next(j_counters, iter);
  }
  return NULL;
}

/*
 * extract the requested integer value out of a counter entry:
 *   - integer entry: the counter itself
 *   - object entry with value-only selector: j_counter[value]
 *   - object entry with name+value selectors: j_counter[name][value]
 */
static oidc_json_t *oidc_metrics_status_select_value(oidc_json_t *j_counter, const char *s_name_param,
                 const char *s_value_param) {
  const oidc_json_t *j_values = NULL;
  if (oidc_json_is_integer(j_counter))
    return j_counter;
  if (s_value_param == NULL)
    return NULL;
  if (s_name_param == NULL)
    return oidc_json_object_get(j_counter, s_value_param);
  j_values = oidc_json_object_get(j_counter, s_name_param);
  if (j_values == NULL)
    return NULL;
  return oidc_json_object_get(j_values, s_value_param);
}

#define OIDC_METRICS_STATUS_PING_KEY "status-ping"

/*
 * readiness variant of format=status (no counter selector): verify that the configured cache
 * backend completes a write/read round-trip and report how provider metadata is configured;
 * returns a 503 with an "ERROR" body when the cache backend is not usable
 */
static int oidc_metrics_status_readiness(request_rec *r) {
  oidc_cfg_t *cfg = ap_get_module_config(r->server->module_config, &auth_openidc_module);
  char *value = NULL;
  const char *msg = NULL;
  const char *s_provider = NULL;
  apr_byte_t ok = FALSE;

  /* write/read round-trip through the configured cache backend */
  if ((oidc_cache_set(r, OIDC_CACHE_SECTION_HEALTH, OIDC_METRICS_STATUS_PING_KEY, "OK",
          apr_time_now() + apr_time_from_sec(60)) == TRUE) &&
      (oidc_cache_get(r, OIDC_CACHE_SECTION_HEALTH, OIDC_METRICS_STATUS_PING_KEY, &value) == TRUE) &&
      (value != NULL) && (_oidc_strcmp(value, "OK") == 0))
    ok = TRUE;

  if (oidc_cfg_metadata_dir_get(cfg) != NULL)
    s_provider = "metadata_dir";
  else if (oidc_cfg_provider_issuer_get(oidc_cfg_provider_get(cfg)) != NULL)
    s_provider = "static";
  else
    s_provider = "none";

  msg = apr_psprintf(r->pool, "%s\ncache: %s: %s\nprovider: metadata: %s\n", ok ? "OK" : "ERROR",
         cfg->cache.impl->name, ok ? "ok" : "fail", s_provider);

  return oidc_util_http_send(r, msg, _oidc_strlen(msg), "text/plain", ok ? OK : HTTP_SERVICE_UNAVAILABLE);
}

/*
 * return status updates
 */
static int oidc_metrics_handle_status(request_rec *r, const char *s_json) {
  const char *msg = "OK\n";
  char *s_metric_param = NULL;
  char *s_server_param = NULL;
  char *s_name_param = NULL;
  char *s_value_param = NULL;
  oidc_json_t *json = NULL;
  const oidc_json_t *j_server = NULL;
  oidc_json_t *j_counters = NULL;
  oidc_json_t *j_counter = NULL;
  const oidc_json_t *j_value = NULL;

  oidc_util_url_parameter_get(r, OIDC_METRICS_SERVER_PARAM, &s_server_param);
  oidc_util_url_parameter_get(r, OIDC_METRICS_COUNTER_PARAM, &s_metric_param);
  oidc_util_url_parameter_get(r, OIDC_METRICS_NAME_PARAM, &s_name_param);
  oidc_util_url_parameter_get(r, OIDC_METRICS_VALUE_PARAM, &s_value_param);

  if (s_server_param == NULL)
    s_server_param = "localhost";

  /* without a counter selector this is a readiness probe rather than a counter query */
  if (s_metric_param == NULL)
    return oidc_metrics_status_readiness(r);

  json = oidc_metrics_json_parse_r(r, s_json);
  if (json == NULL)
    goto end;

  j_server = oidc_json_object_get(json, s_server_param);
  if (j_server == NULL)
    goto end;

  j_counters = oidc_json_object_get(j_server, OIDC_METRICS_COUNTERS);
  if (j_counters == NULL)
    goto end;

  j_counter = oidc_metrics_status_find_counter(r, j_counters, s_metric_param);
  if (j_counter == NULL)
    goto end;

  j_value = oidc_metrics_status_select_value(j_counter, s_name_param, s_value_param);
  if (j_value)
    msg = apr_psprintf(r->pool, "OK: %s\n", _json_int2str(r->pool, oidc_json_integer_value(j_value)));

end:

  if (json)
    oidc_json_decref(json);

  return oidc_util_http_send(r, msg, _oidc_strlen(msg), "text/plain", OK);
}

/*
 * return the bucket table entry for a JSON bucket key
 */
static const oidc_metrics_bucket_t *oidc_metrics_prometheus_bucket_get(const char *json_name) {
  for (int i = 0; i < OIDC_METRICS_BUCKET_NUM; i++)
    if (_oidc_strcmp(_oidc_metric_buckets[i].name, json_name) == 0)
      return &_oidc_metric_buckets[i];
  return NULL;
}

#define OIDC_METRICS_PROMETHEUS_PREFIX "oidc"

/*
 * normalize a metric name to something that Prometheus accepts
 */
static const char *oidc_metric_prometheus_normalize_name(apr_pool_t *pool, const char *name) {
  char *label = apr_psprintf(pool, "%s", name);
  for (size_t i = 0; i < _oidc_strlen(label); i++)
    if (apr_isalnum(label[i]) == 0)
      label[i] = '_';
  return apr_psprintf(pool, "%s_%s", OIDC_METRICS_PROMETHEUS_PREFIX, label);
}

/* Escape a Prometheus text-format label value: backslash, quote and LF are special. */
static const char *oidc_metrics_prometheus_escape_label(apr_pool_t *pool, const char *value) {
  apr_size_t len = _oidc_strlen(value);
  char *result = apr_palloc(pool, (len * 2) + 1);
  char *dst = result;

  for (const char *src = value; *src != '\0'; src++) {
    switch (*src) {
    case '\\':
    case '"':
      *dst++ = '\\';
      *dst++ = *src;
      break;
    case '\n':
      *dst++ = '\\';
      *dst++ = 'n';
      break;
    default:
      *dst++ = *src;
      break;
    }
  }
  *dst = '\0';
  return result;
}

#define OIDC_METRICS_PROMETHEUS_CONTENT_TYPE "text/plain; version=0.0.4"

#define OIDC_METRICS_PROMETHEUS_SERVER "server_name"
#define OIDC_METRICS_PROMETHEUS_BUCKET "bucket"
#define OIDC_METRICS_PROMETHEUS_VALUE "value"
#define OIDC_METRICS_PROMETHEUS_NAME "name"

// loop context for Prometheus output
typedef struct oidc_metric_prometheus_callback_ctx_t {
  char *s_result;
  apr_pool_t *pool;
} oidc_metric_prometheus_callback_ctx_t;

/*
 * append per-(name, value) label-pair lines for a single key under a server's counter
 */
static char *oidc_metrics_prometheus_counter_named(apr_pool_t *pool, char *s_text, const char *s_start,
               const char *s_key, oidc_json_t *j_value) {
  void *iter = oidc_json_object_iter(j_value);
  while (iter) {
    const char *s_value = oidc_json_object_iter_key(iter);
    s_text =
        apr_psprintf(pool, "%s%s,%s=\"%s\",%s=\"%s\"} %s\n", s_text, s_start, OIDC_METRICS_PROMETHEUS_NAME,
         oidc_metrics_prometheus_escape_label(pool, s_key), OIDC_METRICS_PROMETHEUS_VALUE,
         oidc_metrics_prometheus_escape_label(pool, s_value),
         _json_int2str(pool, oidc_json_integer_value(oidc_json_object_iter_value(iter))));
    iter = oidc_json_object_iter_next(j_value, iter);
  }
  return s_text;
}

/*
 * append per-key lines for one server's counter dict; integer entries are value-labeled, sub-dict
 * entries are delegated to oidc_metrics_prometheus_counter_named for name+value labeling
 */
static char *oidc_metrics_prometheus_counter_keyed(apr_pool_t *pool, char *s_text, const char *s_start,
               oidc_json_t *j_counter) {
  void *iter = oidc_json_object_iter(j_counter);
  while (iter) {
    const char *s_key = oidc_json_object_iter_key(iter);
    oidc_json_t *j_value = oidc_json_object_iter_value(iter);
    if (oidc_json_is_integer(j_value))
      s_text =
          apr_psprintf(pool, "%s%s,%s=\"%s\"} %s\n", s_text, s_start, OIDC_METRICS_PROMETHEUS_VALUE,
           oidc_metrics_prometheus_escape_label(pool, s_key),
           _json_int2str(pool, oidc_json_integer_value(j_value)));
    else
      s_text = oidc_metrics_prometheus_counter_named(pool, s_text, s_start, s_key, j_value);
    iter = oidc_json_object_iter_next(j_counter, iter);
  }
  return s_text;
}

/*
 * loop function for converting counter metrics to Prometheus output
 */
static int oidc_metrics_prometheus_counters(oidc_metric_prometheus_callback_ctx_t *ctx, const char *key,
              oidc_json_t *value) {
  oidc_json_t *o_counter = value;
  const oidc_metrics_counter_info_t *info = _oidc_metrics_key2counter_info(key);
  const char *s_label = NULL;
  char *s_text = NULL;

  if (info == NULL)
    return 1;

  s_label = oidc_metric_prometheus_normalize_name(ctx->pool, _oidc_metrics_counter_info2s(ctx->pool, info));
  s_text = apr_psprintf(ctx->pool, "# HELP %s The number of %s.\n", s_label, info->desc);
  s_text = apr_psprintf(ctx->pool, "%s# TYPE %s counter\n", s_text, s_label);

  void *iter = oidc_json_object_iter(o_counter);
  while (iter) {
    const char *s_server = oidc_json_object_iter_key(iter);
    oidc_json_t *j_counter = oidc_json_object_iter_value(iter);
    const char *s_start = apr_psprintf(ctx->pool, "%s{%s=\"%s\"", s_label, OIDC_METRICS_PROMETHEUS_SERVER,
               oidc_metrics_prometheus_escape_label(ctx->pool, s_server));
    if (oidc_json_is_integer(j_counter))
      s_text = apr_psprintf(ctx->pool, "%s%s} %s\n", s_text, s_start,
                _json_int2str(ctx->pool, oidc_json_integer_value(j_counter)));
    else
      s_text = oidc_metrics_prometheus_counter_keyed(ctx->pool, s_text, s_start, j_counter);
    iter = oidc_json_object_iter_next(o_counter, iter);
  }
  ctx->s_result = apr_pstrcat(ctx->pool, ctx->s_result, s_text, "\n", NULL);
  oidc_json_decref(o_counter);
  return 1;
}

/*
 * loop function for converting timing metrics to Prometheus output
 */

static int oidc_metrics_prometheus_timings(oidc_metric_prometheus_callback_ctx_t *ctx, const char *key,
             oidc_json_t *value) {
  const char *s_server = NULL;
  const char *s_key = NULL;
  const oidc_metrics_bucket_t *bucket = NULL;
  oidc_json_t *j_timing = NULL;
  const oidc_json_t *j_member = NULL;
  oidc_json_t *o_timer = value;
  oidc_json_int_t v = 0;
  const oidc_metrics_timing_info_t *info = _oidc_metrics_key2timing_info(key);
  const char *s_label = NULL;
  char *s_text = NULL;
  char *s_secs = NULL;

  if (info == NULL)
    return 1;

  s_label = oidc_metric_prometheus_normalize_name(ctx->pool, _oidc_metrics_timing_info2s(ctx->pool, info));
  /* the unsuffixed millisecond family is deprecated as of 2.4.20.4 in favor of the
   * Prometheus-idiomatic _seconds family emitted alongside it below, and will be
   * removed in a future release */
  s_text = apr_psprintf(ctx->pool, "# HELP %s A histogram of %s.\n", s_label, info->desc);
  s_text = apr_psprintf(ctx->pool, "%s# TYPE %s histogram\n", s_text, s_label);
  s_secs = apr_psprintf(ctx->pool, "# HELP %s_seconds A histogram of %s in seconds.\n", s_label, info->desc);
  s_secs = apr_psprintf(ctx->pool, "%s# TYPE %s_seconds histogram\n", s_secs, s_label);

  void *iter1 = oidc_json_object_iter(o_timer);
  while (iter1) {
    s_server = oidc_json_object_iter_key(iter1);
    const char *s_server_escaped = oidc_metrics_prometheus_escape_label(ctx->pool, s_server);
    j_timing = oidc_json_object_iter_value(iter1);
    void *iter3 = oidc_json_object_iter(j_timing);
    while (iter3) {
      s_key = oidc_json_object_iter_key(iter3);
      j_member = oidc_json_object_iter_value(iter3);
      v = oidc_json_integer_value(j_member);
      bucket = oidc_metrics_prometheus_bucket_get(s_key);
      if (bucket != NULL) {
        s_text = apr_psprintf(ctx->pool, "%s%s_%s{%s,%s=\"%s\"} %s\n", s_text, s_label,
                  OIDC_METRICS_PROMETHEUS_BUCKET, bucket->label,
                  OIDC_METRICS_PROMETHEUS_SERVER, s_server_escaped,
                  _json_int2str(ctx->pool, v));
        s_secs = apr_psprintf(ctx->pool, "%s%s_seconds_%s{%s,%s=\"%s\"} %s\n", s_secs, s_label,
                  OIDC_METRICS_PROMETHEUS_BUCKET, bucket->label_seconds,
                  OIDC_METRICS_PROMETHEUS_SERVER, s_server_escaped,
                  _json_int2str(ctx->pool, v));
      } else if (_oidc_strcmp(s_key, OIDC_METRICS_SUM) == 0) {
        /* the sum is kept in microseconds in shared memory: truncate to
         * milliseconds for the legacy family, exact seconds for _seconds */
        s_text = apr_psprintf(ctx->pool, "%s%s_%s{%s=\"%s\"} %s\n", s_text, s_label, s_key,
                  OIDC_METRICS_PROMETHEUS_SERVER, s_server_escaped,
                  _json_int2str(ctx->pool, v / 1000));
        s_secs = apr_psprintf(ctx->pool, "%s%s_seconds_%s{%s=\"%s\"} %.6f\n", s_secs, s_label,
                  s_key, OIDC_METRICS_PROMETHEUS_SERVER, s_server_escaped,
                  (double)v / 1000000.0);
      } else {
        s_text = apr_psprintf(ctx->pool, "%s%s_%s{%s=\"%s\"} %s\n", s_text, s_label, s_key,
                  OIDC_METRICS_PROMETHEUS_SERVER, s_server_escaped,
                  _json_int2str(ctx->pool, v));
        s_secs = apr_psprintf(ctx->pool, "%s%s_seconds_%s{%s=\"%s\"} %s\n", s_secs, s_label,
                  s_key, OIDC_METRICS_PROMETHEUS_SERVER, s_server_escaped,
                  _json_int2str(ctx->pool, v));
      }
      iter3 = oidc_json_object_iter_next(j_timing, iter3);
    }
    iter1 = oidc_json_object_iter_next(o_timer, iter1);
  }
  ctx->s_result = apr_pstrcat(ctx->pool, ctx->s_result, s_text, "\n", s_secs, "\n", NULL);
  oidc_json_decref(o_timer);
  return 1;
}

/*
 * take a list of metrics from a server indexed list and add it to a type indexed list
 */
static void oidc_metrics_prometheus_convert(apr_hash_t *hash, const char *server, oidc_json_t *list) {
  const char *type = NULL;
  oidc_json_t *src = NULL;
  oidc_json_t *dst = NULL;
  void *iter = oidc_json_object_iter(list);
  while (iter) {
    type = oidc_json_object_iter_key(iter);
    src = oidc_json_object_iter_value(iter);
    dst = (oidc_json_t *)apr_hash_get(hash, type, APR_HASH_KEY_STRING);
    if (dst) {
      oidc_json_object_set(dst, server, src);
    } else {
      dst = oidc_json_object();
      oidc_json_object_set(dst, server, src);
      apr_hash_set(hash, type, APR_HASH_KEY_STRING, dst);
    }
    iter = oidc_json_object_iter_next(list, iter);
  }
}

/*
 * generate output in Prometheus formatting
 */
static int oidc_metrics_handle_prometheus(request_rec *r, const char *s_json) {
  oidc_json_t *json = NULL;
  const oidc_json_t *j_server = NULL;
  const char *s_server = NULL;
  apr_hash_t *t_counters = apr_hash_make(r->pool);
  apr_hash_t *t_timings = apr_hash_make(r->pool);
  apr_hash_index_t *hi = NULL;
  const char *name = NULL;
  void *value = NULL;

  /* start the output with the build info and the metrics subsystem's own health */
  char *s_self = apr_psprintf(
      r->pool,
      "# HELP %s_build_info A metric with a constant '1' value labeled by the module build version.\n"
      "# TYPE %s_build_info gauge\n"
      "%s_build_info{version=\"%s\"} 1\n\n"
      "# HELP %s_metrics_flush_errors The number of metrics flushes dropped because the JSON data outgrew the "
      "shared memory segment.\n"
      "# TYPE %s_metrics_flush_errors counter\n"
      "%s_metrics_flush_errors %u\n\n"
      "# HELP %s_metrics_last_flush_timestamp_seconds The Unix time of the last completed metrics flush cycle in "
      "this server process.\n"
      "# TYPE %s_metrics_last_flush_timestamp_seconds gauge\n"
      "%s_metrics_last_flush_timestamp_seconds %u\n\n",
      OIDC_METRICS_PROMETHEUS_PREFIX, OIDC_METRICS_PROMETHEUS_PREFIX, OIDC_METRICS_PROMETHEUS_PREFIX,
      oidc_metrics_prometheus_escape_label(r->pool, NAMEVERSION), OIDC_METRICS_PROMETHEUS_PREFIX,
      OIDC_METRICS_PROMETHEUS_PREFIX, OIDC_METRICS_PROMETHEUS_PREFIX,
      apr_atomic_read32(&_oidc_metrics_shm_hdr()->flush_errors), OIDC_METRICS_PROMETHEUS_PREFIX,
      OIDC_METRICS_PROMETHEUS_PREFIX, OIDC_METRICS_PROMETHEUS_PREFIX,
      apr_atomic_read32(&_oidc_metrics_shm_hdr()->last_flush));

  oidc_metric_prometheus_callback_ctx_t ctx = {s_self, r->pool};
  void *iter = NULL;

  /* parse the metrics string to JSON */
  json = oidc_metrics_json_parse_r(r, s_json);
  if (json == NULL)
    return OK;

  iter = oidc_json_object_iter(json);
  while (iter) {
    s_server = oidc_json_object_iter_key(iter);
    j_server = oidc_json_object_iter_value(iter);
    oidc_metrics_prometheus_convert(t_counters, s_server,
            oidc_json_object_get(j_server, OIDC_METRICS_COUNTERS));
    oidc_metrics_prometheus_convert(t_timings, s_server,
            oidc_json_object_get(j_server, OIDC_METRICS_TIMINGS));
    iter = oidc_json_object_iter_next(json, iter);
  }

  for (hi = apr_hash_first(r->pool, t_counters); hi; hi = apr_hash_next(hi)) {
    apr_hash_this(hi, (const void **)&name, NULL, &value);
    oidc_metrics_prometheus_counters(&ctx, name, value);
  }

  for (hi = apr_hash_first(r->pool, t_timings); hi; hi = apr_hash_next(hi)) {
    apr_hash_this(hi, (const void **)&name, NULL, &value);
    oidc_metrics_prometheus_timings(&ctx, name, value);
  }

  oidc_json_decref(json);

  return oidc_util_http_send(r, ctx.s_result, _oidc_strlen(ctx.s_result), OIDC_METRICS_PROMETHEUS_CONTENT_TYPE,
           OK);
}

/*
 * definitions for handler callbacks
 */

typedef int (*oidc_metrics_handler_function_t)(request_rec *, const char *);

// holder for output function callback context
typedef struct oidc_metrics_handler_t {
  const char *format;
  oidc_metrics_handler_function_t callback;
  int reset;
} oidc_metrics_content_handler_t;

// output handlers
// NB: no format resets the collected metrics by default (json did before 2.4.20.4);
// a caller that wants the pre-2.4.20.4 behavior passes reset=true explicitly
const oidc_metrics_content_handler_t _oidc_metrics_handlers[] = {
    // first is default
    {"prometheus", oidc_metrics_handle_prometheus, 0},
    {"json", oidc_metrics_handle_json, 0},
    {"internal", oidc_metrics_handle_internal, 0},
    {"status", oidc_metrics_handle_status, 0},
};

#define OIDC_CONTENT_HANDLER_MAX ((int)(sizeof(_oidc_metrics_handlers) / sizeof(oidc_metrics_content_handler_t)))

#define OIDC_METRICS_RESET_PARAM "reset"

/*
 * see if we are going to reset the cache after this
 */
static int oidc_metric_reset(request_rec *r, int dvalue) {
  char *s_reset = NULL;
  int value = 0;

  oidc_util_url_parameter_get(r, OIDC_METRICS_RESET_PARAM, &s_reset);

  if (s_reset == NULL)
    return dvalue;

  if (_oidc_strnatcasecmp(s_reset, "true") == 0)
    value = 1;
  else if (_oidc_strnatcasecmp(s_reset, "false") == 0)
    value = 0;

  return value;
}

#define OIDC_METRICS_FORMAT_PARAM "format"

/*
 * find the format handler
 */
const oidc_metrics_content_handler_t *oidc_metrics_find_handler(request_rec *r) {
  const oidc_metrics_content_handler_t *handler = NULL;
  char *s_format = NULL;

  /* get the specified format */
  oidc_util_url_parameter_get(r, OIDC_METRICS_FORMAT_PARAM, &s_format);

  if (s_format == NULL)
    return &_oidc_metrics_handlers[0];

  for (int i = 0; i < OIDC_CONTENT_HANDLER_MAX; i++) {
    if (_oidc_strcmp(s_format, _oidc_metrics_handlers[i].format) == 0) {
      handler = &_oidc_metrics_handlers[i];
      break;
    }
  }

  if (handler == NULL)
    oidc_warn(r, "could not find a metrics handler for format: %s", s_format);

  return handler;
}

/*
 * return the metrics to the caller and flush the storage
 */
int oidc_metrics_handle_request(request_rec *r) {
  const char *s_json = NULL;
  const oidc_metrics_content_handler_t *handler = NULL;

  /* get the content handler for the format */
  handler = oidc_metrics_find_handler(r);
  if (handler == NULL)
    return HTTP_NOT_FOUND;

  /* lock the global shared memory */
  if (oidc_cache_mutex_lock(r->pool, r->server, _oidc_metrics_global_mutex) == FALSE)
    return HTTP_INTERNAL_SERVER_ERROR;

  /* retrieve the JSON formatted metrics as a string; NB: on the request pool, since
   * it is handed to the content handler below, after the mutex has been released */
  s_json = _oidc_metrics_storage_get(r->server, r->pool);

  /* now that the metrics have been consumed, clear the shared memory segment */
  if (oidc_metric_reset(r, handler->reset))
    oidc_metrics_storage_reset(r->server, r->pool);

  /* unlock the global shared memory */
  if (oidc_cache_mutex_unlock(r->pool, r->server, _oidc_metrics_global_mutex) == FALSE)
    return HTTP_INTERNAL_SERVER_ERROR;

  /* handle the specified format */
  return handler->callback(r, s_json);
}

Coverage Report

Created: 2026-11-12 07:05