/*

 * Copyright (c) 2017 Ericsson AB.

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at:

 *

 *     http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

#ifndef DPIF_NETDEV_PERF_H

#define DPIF_NETDEV_PERF_H 1

#include <stdbool.h>

#include <stddef.h>

#include <stdint.h>

#include <string.h>

#include <time.h>

#include <math.h>

#ifdef DPDK_NETDEV

#include <rte_config.h>

#include <rte_cycles.h>

#endif

#include "openvswitch/vlog.h"

#include "ovs-atomic.h"

#include "timeval.h"

#include "unixctl.h"

#include "util.h"

#ifdef  __cplusplus

extern "C" {

#endif

/* This module encapsulates data structures and functions to maintain basic PMD

 * performance metrics such as packet counters, execution cycles as well as

 * histograms and time series recording for more detailed PMD metrics.

 *

 * It provides a clean API for dpif-netdev to initialize, update and read and

 * reset these metrics.

 *

 * The basic set of PMD counters is implemented as atomic_uint64_t variables

 * to guarantee correct read also in 32-bit systems.

 *

 * The detailed PMD performance metrics are only supported on 64-bit systems

 * with atomic 64-bit read and store semantics for plain uint64_t counters.

 */

/* Set of counter types maintained in pmd_perf_stats. */

enum pmd_stat_type {

    PMD_STAT_PHWOL_HIT,     /* Packets that had a partial HWOL hit (phwol). */

    PMD_STAT_SIMPLE_HIT,    /* Packets that had a simple match hit. */

    PMD_STAT_EXACT_HIT,     /* Packets that had an exact match (emc). */

    PMD_STAT_SMC_HIT,       /* Packets that had a sig match hit (SMC). */

    PMD_STAT_MASKED_HIT,    /* Packets that matched in the flow table. */

    PMD_STAT_MISS,          /* Packets that did not match and upcall was ok. */

    PMD_STAT_LOST,          /* Packets that did not match and upcall failed. */

                            /* The above statistics account for the total

                             * number of packet passes through the datapath

                             * pipeline and should not be overlapping with each

                             * other. */

    PMD_STAT_MASKED_LOOKUP, /* Number of subtable lookups for flow table

                               hits. Each MASKED_HIT hit will have >= 1

                               MASKED_LOOKUP(s). */

    PMD_STAT_RECV,          /* Packets entering the datapath pipeline from an

                             * interface. */

    PMD_STAT_RECIRC,        /* Packets reentering the datapath pipeline due to

                             * recirculation. */

    PMD_STAT_SENT_PKTS,     /* Packets that have been sent. */

    PMD_STAT_SENT_BATCHES,  /* Number of batches sent. */

    PMD_CYCLES_ITER_IDLE,   /* Cycles spent in idle iterations. */

    PMD_CYCLES_ITER_BUSY,   /* Cycles spent in busy iterations. */

    PMD_CYCLES_UPCALL,      /* Cycles spent processing upcalls. */

    PMD_SLEEP_ITER,         /* Iterations where a sleep has taken place. */

    PMD_CYCLES_SLEEP,       /* Total cycles slept to save power. */

    PMD_N_STATS

};

/* Array of PMD counters indexed by enum pmd_stat_type.

 * The n[] array contains the actual counter values since initialization

 * of the PMD. Counters are atomically updated from the PMD but are

 * read and cleared also from other processes. To clear the counters at

 * PMD run-time, the current counter values are copied over to the zero[]

 * array. To read counters we subtract zero[] value from n[]. */

struct pmd_counters {

    atomic_uint64_t n[PMD_N_STATS];     /* Value since _init(). */

    uint64_t zero[PMD_N_STATS];         /* Value at last _clear().  */

};

/* Data structure to collect statistical distribution of an integer measurement

 * type in form of a histogram. The wall[] array contains the inclusive

 * upper boundaries of the bins, while the bin[] array contains the actual

 * counters per bin. The histogram walls are typically set automatically

 * using the functions provided below.*/

#define NUM_BINS 32             /* Number of histogram bins. */

struct histogram {

    uint32_t wall[NUM_BINS];

    uint64_t bin[NUM_BINS];

};

/* Data structure to record details PMD execution metrics per iteration for

 * a history period of up to HISTORY_LEN iterations in circular buffer.

 * Also used to record up to HISTORY_LEN millisecond averages/totals of these

 * metrics.*/

struct iter_stats {

    uint64_t timestamp;         /* Iteration no. or millisecond. */

    uint64_t cycles;            /* Number of TSC cycles spent in it. or ms. */

    uint64_t busy_cycles;       /* Cycles spent in busy iterations or ms. */

    uint32_t iterations;        /* Iterations in ms. */

    uint32_t pkts;              /* Packets processed in iteration or ms. */

    uint32_t upcalls;           /* Number of upcalls in iteration or ms. */

    uint32_t upcall_cycles;     /* Cycles spent in upcalls in it. or ms. */

    uint32_t batches;           /* Number of rx batches in iteration or ms. */

    uint32_t max_vhost_qfill;   /* Maximum fill level in iteration or ms. */

};

#define HISTORY_LEN 1000        /* Length of recorded history

                                   (iterations and ms). */

#define DEF_HIST_SHOW 20        /* Default number of history samples to

                                   display. */

struct history {

    size_t idx;                 /* Slot to which next call to history_store()

                                   will write. */

    struct iter_stats sample[HISTORY_LEN];

};

/* Container for all performance metrics of a PMD within the struct

 * dp_netdev_pmd_thread. The metrics must be updated from within the PMD

 * thread but can be read from any thread. The basic PMD counters in

 * struct pmd_counters can be read without protection against concurrent

 * clearing. The other metrics may only be safely read with the clear_mutex

 * held to protect against concurrent clearing. */

struct pmd_perf_stats {

    /* Prevents interference between PMD polling and stats clearing. */

    struct ovs_mutex stats_mutex;

    /* Set by CLI thread to order clearing of PMD stats. */

    volatile bool clear;

    /* Prevents stats retrieval while clearing is in progress. */

    struct ovs_mutex clear_mutex;

    /* Start of the current performance measurement period. */

    uint64_t start_ms;

    /* Counter for PMD iterations. */

    uint64_t iteration_cnt;

    /* Start of the current iteration. */

    uint64_t start_tsc;

    /* Latest TSC time stamp taken in PMD. */

    uint64_t last_tsc;

    /* Used to space certain checks in time. */

    uint64_t next_check_tsc;

    /* If non-NULL, outermost cycle timer currently running in PMD. */

    struct cycle_timer *cur_timer;

    /* Set of PMD counters with their zero offsets. */

    struct pmd_counters counters;

    /* Statistics of the current iteration. */

    struct iter_stats current;

    /* Totals for the current millisecond. */

    struct iter_stats totals;

    /* Histograms for the PMD metrics. */

    struct histogram cycles;

    struct histogram pkts;

    struct histogram cycles_per_pkt;

    struct histogram upcalls;

    struct histogram cycles_per_upcall;

    struct histogram pkts_per_batch;

    struct histogram max_vhost_qfill;

    /* Iteration history buffer. */

    struct history iterations;

    /* Millisecond history buffer. */

    struct history milliseconds;

    /* Suspicious iteration log. */

    uint32_t log_susp_it;

    /* Start of iteration range to log. */

    uint32_t log_begin_it;

    /* End of iteration range to log. */

    uint32_t log_end_it;

    /* Reason for logging suspicious iteration. */

    char *log_reason;

};

#ifdef __linux__

static inline uint64_t

rdtsc_syscall(struct pmd_perf_stats *s)

{

    struct timespec val;

    uint64_t v;

    if (clock_gettime(CLOCK_MONOTONIC_RAW, &val) != 0) {

       return s->last_tsc;

    }

    v  = val.tv_sec * UINT64_C(1000000000) + val.tv_nsec;

    return s->last_tsc = v;

}

Unexecuted instantiation: dpif-netdev.c:rdtsc_syscall

Unexecuted instantiation: dpif-netdev-dpcls.c:rdtsc_syscall

Unexecuted instantiation: dpif-netdev-perf.c:rdtsc_syscall

#endif

/* Support for accurate timing of PMD execution on TSC clock cycle level.

 * These functions are intended to be invoked in the context of pmd threads. */

/* Read the TSC cycle register and cache it. Any function not requiring clock

 * cycle accuracy should read the cached value using cycles_counter_get() to

 * avoid the overhead of reading the TSC register. */

static inline uint64_t

cycles_counter_update(struct pmd_perf_stats *s)

{

#ifdef DPDK_NETDEV

    return s->last_tsc = rte_get_tsc_cycles();

#elif defined(__x86_64__)

    uint32_t h, l;

    asm volatile("rdtsc" : "=a" (l), "=d" (h));

    return s->last_tsc = ((uint64_t) h << 32) | l;

#elif defined(__aarch64__)

    asm volatile("mrs %0, cntvct_el0" : "=r" (s->last_tsc));

    return s->last_tsc;

#elif defined(__linux__)

    return rdtsc_syscall(s);

#else

    return s->last_tsc = 0;

#endif

}

Unexecuted instantiation: dpif-netdev.c:cycles_counter_update

Unexecuted instantiation: dpif-netdev-dpcls.c:cycles_counter_update

Unexecuted instantiation: dpif-netdev-perf.c:cycles_counter_update

static inline uint64_t

cycles_counter_get(struct pmd_perf_stats *s)

{

    return s->last_tsc;

}

Unexecuted instantiation: dpif-netdev.c:cycles_counter_get

Unexecuted instantiation: dpif-netdev-dpcls.c:cycles_counter_get

Unexecuted instantiation: dpif-netdev-perf.c:cycles_counter_get

void pmd_perf_estimate_tsc_frequency(void);

/* A nestable timer for measuring execution time in TSC cycles.

 *

 * Usage:

 * struct cycle_timer timer;

 *

 * cycle_timer_start(pmd, &timer);

 * <Timed execution>

 * uint64_t cycles = cycle_timer_stop(pmd, &timer);

 *

 * The caller must guarantee that a call to cycle_timer_start() is always

 * paired with a call to cycle_stimer_stop().

 *

 * Is is possible to have nested cycles timers within the timed code. The

 * execution time measured by the nested timers is excluded from the time

 * measured by the embracing timer.

 */

struct cycle_timer {

    uint64_t start;

    uint64_t suspended;

    struct cycle_timer *interrupted;

};

static inline void

cycle_timer_start(struct pmd_perf_stats *s,

                  struct cycle_timer *timer)

{

    struct cycle_timer *cur_timer = s->cur_timer;

    uint64_t now = cycles_counter_update(s);

    if (cur_timer) {

        cur_timer->suspended = now;

    }

    timer->interrupted = cur_timer;

    timer->start = now;

    timer->suspended = 0;

    s->cur_timer = timer;

}

Unexecuted instantiation: dpif-netdev.c:cycle_timer_start

Unexecuted instantiation: dpif-netdev-dpcls.c:cycle_timer_start

Unexecuted instantiation: dpif-netdev-perf.c:cycle_timer_start

static inline uint64_t

cycle_timer_stop(struct pmd_perf_stats *s,

                 struct cycle_timer *timer)

{

    /* Assert that this is the current cycle timer. */

    ovs_assert(s->cur_timer == timer);

    uint64_t now = cycles_counter_update(s);

    struct cycle_timer *intr_timer = timer->interrupted;

    if (intr_timer) {

        /* Adjust the start offset by the suspended cycles. */

        intr_timer->start += now - intr_timer->suspended;

    }

    /* Restore suspended timer, if any. */

    s->cur_timer = intr_timer;

    return now - timer->start;

}

Unexecuted instantiation: dpif-netdev.c:cycle_timer_stop

Unexecuted instantiation: dpif-netdev-dpcls.c:cycle_timer_stop

Unexecuted instantiation: dpif-netdev-perf.c:cycle_timer_stop

/* Functions to initialize and reset the PMD performance metrics. */

void pmd_perf_stats_init(struct pmd_perf_stats *s);

void pmd_perf_stats_clear(struct pmd_perf_stats *s);

void pmd_perf_stats_clear_lock(struct pmd_perf_stats *s);

/* Functions to read and update PMD counters. */

void pmd_perf_read_counters(struct pmd_perf_stats *s,

                            uint64_t stats[PMD_N_STATS]);

/* PMD performance counters are updated lock-less. For real PMDs

 * they are only updated from the PMD thread itself. In the case of the

 * NON-PMD they might be updated from multiple threads, but we can live

 * with losing a rare update as 100% accuracy is not required.

 * However, as counters are read for display from outside the PMD thread

 * with e.g. pmd-stats-show, we make sure that the 64-bit read and store

 * operations are atomic also on 32-bit systems so that readers cannot

 * not read garbage. On 64-bit systems this incurs no overhead. */

static inline void

pmd_perf_update_counter(struct pmd_perf_stats *s,

                        enum pmd_stat_type counter, int delta)

{

    uint64_t tmp;

    atomic_read_relaxed(&s->counters.n[counter], &tmp);

    tmp += delta;

    atomic_store_relaxed(&s->counters.n[counter], tmp);

}

Unexecuted instantiation: dpif-netdev.c:pmd_perf_update_counter

Unexecuted instantiation: dpif-netdev-dpcls.c:pmd_perf_update_counter

Unexecuted instantiation: dpif-netdev-perf.c:pmd_perf_update_counter

/* Functions to manipulate a sample history. */

static inline void

histogram_add_sample(struct histogram *hist, uint32_t val)

{

    /* TODO: Can do better with binary search? */

    for (int i = 0; i < NUM_BINS-1; i++) {

        if (val <= hist->wall[i]) {

            hist->bin[i]++;

            return;

        }

    }

    hist->bin[NUM_BINS-1]++;

}

Unexecuted instantiation: dpif-netdev.c:histogram_add_sample

Unexecuted instantiation: dpif-netdev-dpcls.c:histogram_add_sample

Unexecuted instantiation: dpif-netdev-perf.c:histogram_add_sample

uint64_t histogram_samples(const struct histogram *hist);

/* This function is used to advance the given history index by positive

 * offset in the circular history buffer. */

static inline uint32_t

history_add(uint32_t idx, uint32_t offset)

{

    return (idx + offset) % HISTORY_LEN;

}

Unexecuted instantiation: dpif-netdev.c:history_add

Unexecuted instantiation: dpif-netdev-dpcls.c:history_add

Unexecuted instantiation: dpif-netdev-perf.c:history_add

/* This function computes the difference between two indices into the

 * circular history buffer. The result is always positive in the range

 * 0 .. HISTORY_LEN-1 and specifies the number of steps to reach idx1

 * starting from idx2. It can also be used to retreat the history index

 * idx1 by idx2 steps. */

static inline uint32_t

history_sub(uint32_t idx1, uint32_t idx2)

{

    return (idx1 + HISTORY_LEN - idx2) % HISTORY_LEN;

}

Unexecuted instantiation: dpif-netdev.c:history_sub

Unexecuted instantiation: dpif-netdev-dpcls.c:history_sub

Unexecuted instantiation: dpif-netdev-perf.c:history_sub

static inline struct iter_stats *

history_current(struct history *h)

{

    return &h->sample[h->idx];

}

Unexecuted instantiation: dpif-netdev.c:history_current

Unexecuted instantiation: dpif-netdev-dpcls.c:history_current

Unexecuted instantiation: dpif-netdev-perf.c:history_current

static inline struct iter_stats *

history_next(struct history *h)

{

    size_t next_idx = history_add(h->idx, 1);

    struct iter_stats *next = &h->sample[next_idx];

    memset(next, 0, sizeof(*next));

    h->idx = next_idx;

    return next;

}

Unexecuted instantiation: dpif-netdev.c:history_next

Unexecuted instantiation: dpif-netdev-dpcls.c:history_next

Unexecuted instantiation: dpif-netdev-perf.c:history_next

static inline struct iter_stats *

history_store(struct history *h, struct iter_stats *is)

{

    if (is) {

        h->sample[h->idx] = *is;

    }

    /* Advance the history pointer */

    return history_next(h);

}

Unexecuted instantiation: dpif-netdev.c:history_store

Unexecuted instantiation: dpif-netdev-dpcls.c:history_store

Unexecuted instantiation: dpif-netdev-perf.c:history_store

/* Data and function related to logging of suspicious iterations. */

extern bool log_enabled;

extern bool log_extend;

extern uint32_t log_q_thr;

extern uint64_t iter_cycle_threshold;

void pmd_perf_set_log_susp_iteration(struct pmd_perf_stats *s, char *reason);

void pmd_perf_log_susp_iteration_neighborhood(struct pmd_perf_stats *s);

/* Functions recording PMD metrics per iteration. */

void

pmd_perf_start_iteration(struct pmd_perf_stats *s);

void

pmd_perf_end_iteration(struct pmd_perf_stats *s, int rx_packets,

                       int tx_packets, uint64_t sleep_cycles,

                       bool full_metrics);

/* Formatting the output of commands. */

struct pmd_perf_params {

    int command_type;

    bool histograms;

    size_t iter_hist_len;

    size_t ms_hist_len;

};

void pmd_perf_format_overall_stats(struct ds *str, struct pmd_perf_stats *s,

                                   double duration);

void pmd_perf_format_histograms(struct ds *str, struct pmd_perf_stats *s);

void pmd_perf_format_iteration_history(struct ds *str,

                                       struct pmd_perf_stats *s,

                                       int n_iter);

void pmd_perf_format_ms_history(struct ds *str, struct pmd_perf_stats *s,

                                int n_ms);

void pmd_perf_log_set_cmd(struct unixctl_conn *conn,

                          int argc, const char *argv[],

                          void *aux OVS_UNUSED);

#ifdef  __cplusplus

}

#endif

#endif /* DPIF_NETDEV_PERF_H */