/* SPDX-License-Identifier: GPL-2.0 */
      /*
       * connection tracking helpers.
       *
       * 16 Dec 2003: Yasuyuki Kozakai @USAGI <yasuyuki.kozakai@toshiba.co.jp>
       *        - generalize L3 protocol dependent part.
       *
       * Derived from include/linux/netfiter_ipv4/ip_conntrack_helper.h
       */
      
      #ifndef _NF_CONNTRACK_HELPER_H
      #define _NF_CONNTRACK_HELPER_H
      #include <linux/refcount.h>
      #include <net/netfilter/nf_conntrack.h>
      #include <net/netfilter/nf_conntrack_extend.h>
      #include <net/netfilter/nf_conntrack_expect.h>
      
      #define NF_NAT_HELPER_PREFIX                "ip_nat_"
      #define NF_NAT_HELPER_NAME(name)        NF_NAT_HELPER_PREFIX name
      #define MODULE_ALIAS_NF_NAT_HELPER(name) \
              MODULE_ALIAS(NF_NAT_HELPER_NAME(name))
      
      struct module;
      
      enum nf_ct_helper_flags {
              NF_CT_HELPER_F_USERSPACE        = (1 << 0),
              NF_CT_HELPER_F_CONFIGURED        = (1 << 1),
      };
      
      #define NF_CT_HELPER_NAME_LEN        16
      
      struct nf_conntrack_helper {
              struct hlist_node hnode;        /* Internal use. */
      
              char name[NF_CT_HELPER_NAME_LEN]; /* name of the module */
              refcount_t refcnt;
              struct module *me;                /* pointer to self */
              const struct nf_conntrack_expect_policy *expect_policy;
      
              /* Tuple of things we will help (compared against server response) */
              struct nf_conntrack_tuple tuple;
      
              /* Function to call when data passes; return verdict, or -1 to
                 invalidate. */
              int (*help)(struct sk_buff *skb,
                          unsigned int protoff,
                          struct nf_conn *ct,
                          enum ip_conntrack_info conntrackinfo);
      
              void (*destroy)(struct nf_conn *ct);
      
              int (*from_nlattr)(struct nlattr *attr, struct nf_conn *ct);
              int (*to_nlattr)(struct sk_buff *skb, const struct nf_conn *ct);
              unsigned int expect_class_max;
      
              unsigned int flags;
      
              /* For user-space helpers: */
              unsigned int queue_num;
              /* length of userspace private data stored in nf_conn_help->data */
              u16 data_len;
              /* name of NAT helper module */
              char nat_mod_name[NF_CT_HELPER_NAME_LEN];
      };
      
      /* Must be kept in sync with the classes defined by helpers */
      #define NF_CT_MAX_EXPECT_CLASSES        4
      
      /* nf_conn feature for connections that have a helper */
      struct nf_conn_help {
              /* Helper. if any */
              struct nf_conntrack_helper __rcu *helper;
      
              struct hlist_head expectations;
      
              /* Current number of expected connections */
              u8 expecting[NF_CT_MAX_EXPECT_CLASSES];
      
              /* private helper information. */
              char data[32] __aligned(8);
      };
      
      #define NF_CT_HELPER_BUILD_BUG_ON(structsize) \
              BUILD_BUG_ON((structsize) > FIELD_SIZEOF(struct nf_conn_help, data))
      
      struct nf_conntrack_helper *__nf_conntrack_helper_find(const char *name,
                                                             u16 l3num, u8 protonum);
      
      struct nf_conntrack_helper *nf_conntrack_helper_try_module_get(const char *name,
                                                                     u16 l3num,
                                                                     u8 protonum);
      void nf_conntrack_helper_put(struct nf_conntrack_helper *helper);
      
      void nf_ct_helper_init(struct nf_conntrack_helper *helper,
                             u16 l3num, u16 protonum, const char *name,
                             u16 default_port, u16 spec_port, u32 id,
                             const struct nf_conntrack_expect_policy *exp_pol,
                             u32 expect_class_max,
                             int (*help)(struct sk_buff *skb, unsigned int protoff,
                                         struct nf_conn *ct,
                                         enum ip_conntrack_info ctinfo),
                             int (*from_nlattr)(struct nlattr *attr,
                                                struct nf_conn *ct),
                             struct module *module);
      
      int nf_conntrack_helper_register(struct nf_conntrack_helper *);
      void nf_conntrack_helper_unregister(struct nf_conntrack_helper *);
      
      int nf_conntrack_helpers_register(struct nf_conntrack_helper *, unsigned int);
      void nf_conntrack_helpers_unregister(struct nf_conntrack_helper *,
                                           unsigned int);
      
      struct nf_conn_help *nf_ct_helper_ext_add(struct nf_conn *ct, gfp_t gfp);
      
      int __nf_ct_try_assign_helper(struct nf_conn *ct, struct nf_conn *tmpl,
                                    gfp_t flags);
      
      void nf_ct_helper_destroy(struct nf_conn *ct);
      
      static inline struct nf_conn_help *nfct_help(const struct nf_conn *ct)
      {
 1690         return nf_ct_ext_find(ct, NF_CT_EXT_HELPER);
      }
      
      static inline void *nfct_help_data(const struct nf_conn *ct)
      {
              struct nf_conn_help *help;
      
              help = nf_ct_ext_find(ct, NF_CT_EXT_HELPER);
      
              return (void *)help->data;
      }
      
      void nf_conntrack_helper_pernet_init(struct net *net);
      
      int nf_conntrack_helper_init(void);
      void nf_conntrack_helper_fini(void);
      
      int nf_conntrack_broadcast_help(struct sk_buff *skb, struct nf_conn *ct,
                                      enum ip_conntrack_info ctinfo,
                                      unsigned int timeout);
      
      struct nf_ct_helper_expectfn {
              struct list_head head;
              const char *name;
              void (*expectfn)(struct nf_conn *ct, struct nf_conntrack_expect *exp);
      };
      
      __printf(3,4)
      void nf_ct_helper_log(struct sk_buff *skb, const struct nf_conn *ct,
                            const char *fmt, ...);
      
      void nf_ct_helper_expectfn_register(struct nf_ct_helper_expectfn *n);
      void nf_ct_helper_expectfn_unregister(struct nf_ct_helper_expectfn *n);
      struct nf_ct_helper_expectfn *
      nf_ct_helper_expectfn_find_by_name(const char *name);
      struct nf_ct_helper_expectfn *
      nf_ct_helper_expectfn_find_by_symbol(const void *symbol);
      
      extern struct hlist_head *nf_ct_helper_hash;
      extern unsigned int nf_ct_helper_hsize;
      
      struct nf_conntrack_nat_helper {
              struct list_head list;
              char mod_name[NF_CT_HELPER_NAME_LEN];        /* module name */
              struct module *module;                        /* pointer to self */
      };
      
      #define NF_CT_NAT_HELPER_INIT(name) \
              { \
              .mod_name = NF_NAT_HELPER_NAME(name), \
              .module = THIS_MODULE \
              }
      
      void nf_nat_helper_register(struct nf_conntrack_nat_helper *nat);
      void nf_nat_helper_unregister(struct nf_conntrack_nat_helper *nat);
      int nf_nat_helper_try_module_get(const char *name, u16 l3num,
                                       u8 protonum);
      void nf_nat_helper_put(struct nf_conntrack_helper *helper);
      #endif /*_NF_CONNTRACK_HELPER_H*/
      // SPDX-License-Identifier: GPL-2.0-or-later
      /*
       *  net/dccp/input.c
       *
       *  An implementation of the DCCP protocol
       *  Arnaldo Carvalho de Melo <acme@conectiva.com.br>
       */
      
      #include <linux/dccp.h>
      #include <linux/skbuff.h>
      #include <linux/slab.h>
      
      #include <net/sock.h>
      
      #include "ackvec.h"
      #include "ccid.h"
      #include "dccp.h"
      
      /* rate-limit for syncs in reply to sequence-invalid packets; RFC 4340, 7.5.4 */
      int sysctl_dccp_sync_ratelimit        __read_mostly = HZ / 8;
      
      static void dccp_enqueue_skb(struct sock *sk, struct sk_buff *skb)
      {
   14         __skb_pull(skb, dccp_hdr(skb)->dccph_doff * 4);
              __skb_queue_tail(&sk->sk_receive_queue, skb);
   14         skb_set_owner_r(skb, sk);
   14         sk->sk_data_ready(sk);
      }
      
      static void dccp_fin(struct sock *sk, struct sk_buff *skb)
      {
              /*
               * On receiving Close/CloseReq, both RD/WR shutdown are performed.
               * RFC 4340, 8.3 says that we MAY send further Data/DataAcks after
               * receiving the closing segment, but there is no guarantee that such
               * data will be processed at all.
               */
    9         sk->sk_shutdown = SHUTDOWN_MASK;
              sock_set_flag(sk, SOCK_DONE);
              dccp_enqueue_skb(sk, skb);
      }
      
    2 static int dccp_rcv_close(struct sock *sk, struct sk_buff *skb)
      {
              int queued = 0;
      
    2         switch (sk->sk_state) {
              /*
               * We ignore Close when received in one of the following states:
               *  - CLOSED                (may be a late or duplicate packet)
               *  - PASSIVE_CLOSEREQ        (the peer has sent a CloseReq earlier)
               *  - RESPOND                (already handled by dccp_check_req)
               */
              case DCCP_CLOSING:
                      /*
                       * Simultaneous-close: receiving a Close after sending one. This
                       * can happen if both client and server perform active-close and
                       * will result in an endless ping-pong of crossing and retrans-
                       * mitted Close packets, which only terminates when one of the
                       * nodes times out (min. 64 seconds). Quicker convergence can be
                       * achieved when one of the nodes acts as tie-breaker.
                       * This is ok as both ends are done with data transfer and each
                       * end is just waiting for the other to acknowledge termination.
                       */
                      if (dccp_sk(sk)->dccps_role != DCCP_ROLE_CLIENT)
                              break;
                      /* fall through */
              case DCCP_REQUESTING:
              case DCCP_ACTIVE_CLOSEREQ:
    2                 dccp_send_reset(sk, DCCP_RESET_CODE_CLOSED);
                      dccp_done(sk);
                      break;
              case DCCP_OPEN:
              case DCCP_PARTOPEN:
                      /* Give waiting application a chance to read pending data */
                      queued = 1;
                      dccp_fin(sk, skb);
                      dccp_set_state(sk, DCCP_PASSIVE_CLOSE);
                      /* fall through */
              case DCCP_PASSIVE_CLOSE:
                      /*
                       * Retransmitted Close: we have already enqueued the first one.
                       */
                      sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP);
              }
    2         return queued;
      }
      
      static int dccp_rcv_closereq(struct sock *sk, struct sk_buff *skb)
      {
              int queued = 0;
      
              /*
               *   Step 7: Check for unexpected packet types
               *      If (S.is_server and P.type == CloseReq)
               *          Send Sync packet acknowledging P.seqno
               *          Drop packet and return
               */
              if (dccp_sk(sk)->dccps_role != DCCP_ROLE_CLIENT) {
                      dccp_send_sync(sk, DCCP_SKB_CB(skb)->dccpd_seq, DCCP_PKT_SYNC);
                      return queued;
              }
      
              /* Step 13: process relevant Client states < CLOSEREQ */
              switch (sk->sk_state) {
              case DCCP_REQUESTING:
                      dccp_send_close(sk, 0);
                      dccp_set_state(sk, DCCP_CLOSING);
                      break;
              case DCCP_OPEN:
              case DCCP_PARTOPEN:
                      /* Give waiting application a chance to read pending data */
                      queued = 1;
                      dccp_fin(sk, skb);
                      dccp_set_state(sk, DCCP_PASSIVE_CLOSEREQ);
                      /* fall through */
              case DCCP_PASSIVE_CLOSEREQ:
                      sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP);
              }
              return queued;
      }
      
      static u16 dccp_reset_code_convert(const u8 code)
      {
              static const u16 error_code[] = {
              [DCCP_RESET_CODE_CLOSED]             = 0,        /* normal termination */
              [DCCP_RESET_CODE_UNSPECIFIED]             = 0,        /* nothing known */
              [DCCP_RESET_CODE_ABORTED]             = ECONNRESET,
      
              [DCCP_RESET_CODE_NO_CONNECTION]             = ECONNREFUSED,
              [DCCP_RESET_CODE_CONNECTION_REFUSED] = ECONNREFUSED,
              [DCCP_RESET_CODE_TOO_BUSY]             = EUSERS,
              [DCCP_RESET_CODE_AGGRESSION_PENALTY] = EDQUOT,
      
              [DCCP_RESET_CODE_PACKET_ERROR]             = ENOMSG,
              [DCCP_RESET_CODE_BAD_INIT_COOKIE]    = EBADR,
              [DCCP_RESET_CODE_BAD_SERVICE_CODE]   = EBADRQC,
              [DCCP_RESET_CODE_OPTION_ERROR]             = EILSEQ,
              [DCCP_RESET_CODE_MANDATORY_ERROR]    = EOPNOTSUPP,
              };
      
              return code >= DCCP_MAX_RESET_CODES ? 0 : error_code[code];
      }
      
      static void dccp_rcv_reset(struct sock *sk, struct sk_buff *skb)
      {
    9         u16 err = dccp_reset_code_convert(dccp_hdr_reset(skb)->dccph_reset_code);
      
              sk->sk_err = err;
      
              /* Queue the equivalent of TCP fin so that dccp_recvmsg exits the loop */
              dccp_fin(sk, skb);
      
    6         if (err && !sock_flag(sk, SOCK_DEAD))
    6                 sk_wake_async(sk, SOCK_WAKE_IO, POLL_ERR);
    9         dccp_time_wait(sk, DCCP_TIME_WAIT, 0);
      }
      
      static void dccp_handle_ackvec_processing(struct sock *sk, struct sk_buff *skb)
      {
   53         struct dccp_ackvec *av = dccp_sk(sk)->dccps_hc_rx_ackvec;
      
              if (av == NULL)
                      return;
   52         if (DCCP_SKB_CB(skb)->dccpd_ack_seq != DCCP_PKT_WITHOUT_ACK_SEQ)
   52                 dccp_ackvec_clear_state(av, DCCP_SKB_CB(skb)->dccpd_ack_seq);
   53         dccp_ackvec_input(av, skb);
      }
      
      static void dccp_deliver_input_to_ccids(struct sock *sk, struct sk_buff *skb)
      {
              const struct dccp_sock *dp = dccp_sk(sk);
      
              /* Don't deliver to RX CCID when node has shut down read end. */
   53         if (!(sk->sk_shutdown & RCV_SHUTDOWN))
   52                 ccid_hc_rx_packet_recv(dp->dccps_hc_rx_ccid, sk, skb);
              /*
               * Until the TX queue has been drained, we can not honour SHUT_WR, since
               * we need received feedback as input to adjust congestion control.
               */
   53         if (sk->sk_write_queue.qlen > 0 || !(sk->sk_shutdown & SEND_SHUTDOWN))
   53                 ccid_hc_tx_packet_recv(dp->dccps_hc_tx_ccid, sk, skb);
   53 }
      
      static int dccp_check_seqno(struct sock *sk, struct sk_buff *skb)
      {
   57         const struct dccp_hdr *dh = dccp_hdr(skb);
              struct dccp_sock *dp = dccp_sk(sk);
              u64 lswl, lawl, seqno = DCCP_SKB_CB(skb)->dccpd_seq,
                              ackno = DCCP_SKB_CB(skb)->dccpd_ack_seq;
      
              /*
               *   Step 5: Prepare sequence numbers for Sync
               *     If P.type == Sync or P.type == SyncAck,
               *          If S.AWL <= P.ackno <= S.AWH and P.seqno >= S.SWL,
               *             / * P is valid, so update sequence number variables
               *                 accordingly.  After this update, P will pass the tests
               *                 in Step 6.  A SyncAck is generated if necessary in
               *                 Step 15 * /
               *             Update S.GSR, S.SWL, S.SWH
               *          Otherwise,
               *             Drop packet and return
               */
   57         if (dh->dccph_type == DCCP_PKT_SYNC ||
                  dh->dccph_type == DCCP_PKT_SYNCACK) {
                      if (between48(ackno, dp->dccps_awl, dp->dccps_awh) &&
                          dccp_delta_seqno(dp->dccps_swl, seqno) >= 0)
                              dccp_update_gsr(sk, seqno);
                      else
                              return -1;
              }
      
              /*
               *   Step 6: Check sequence numbers
               *      Let LSWL = S.SWL and LAWL = S.AWL
               *      If P.type == CloseReq or P.type == Close or P.type == Reset,
               *          LSWL := S.GSR + 1, LAWL := S.GAR
               *      If LSWL <= P.seqno <= S.SWH
               *             and (P.ackno does not exist or LAWL <= P.ackno <= S.AWH),
               *          Update S.GSR, S.SWL, S.SWH
               *          If P.type != Sync,
               *             Update S.GAR
               */
              lswl = dp->dccps_swl;
              lawl = dp->dccps_awl;
      
   57         if (dh->dccph_type == DCCP_PKT_CLOSEREQ ||
                  dh->dccph_type == DCCP_PKT_CLOSE ||
                  dh->dccph_type == DCCP_PKT_RESET) {
    3                 lswl = ADD48(dp->dccps_gsr, 1);
                      lawl = dp->dccps_gar;
              }
      
   57         if (between48(seqno, lswl, dp->dccps_swh) &&
                  (ackno == DCCP_PKT_WITHOUT_ACK_SEQ ||
   57              between48(ackno, lawl, dp->dccps_awh))) {
   57                 dccp_update_gsr(sk, seqno);
      
   57                 if (dh->dccph_type != DCCP_PKT_SYNC &&
                          ackno != DCCP_PKT_WITHOUT_ACK_SEQ &&
   57                     after48(ackno, dp->dccps_gar))
   57                         dp->dccps_gar = ackno;
              } else {
                      unsigned long now = jiffies;
                      /*
                       *   Step 6: Check sequence numbers
                       *      Otherwise,
                       *         If P.type == Reset,
                       *            Send Sync packet acknowledging S.GSR
                       *         Otherwise,
                       *            Send Sync packet acknowledging P.seqno
                       *      Drop packet and return
                       *
                       *   These Syncs are rate-limited as per RFC 4340, 7.5.4:
                       *   at most 1 / (dccp_sync_rate_limit * HZ) Syncs per second.
                       */
                      if (time_before(now, (dp->dccps_rate_last +
                                            sysctl_dccp_sync_ratelimit)))
                              return -1;
      
                      DCCP_WARN("Step 6 failed for %s packet, "
                                "(LSWL(%llu) <= P.seqno(%llu) <= S.SWH(%llu)) and "
                                "(P.ackno %s or LAWL(%llu) <= P.ackno(%llu) <= S.AWH(%llu), "
                                "sending SYNC...\n",  dccp_packet_name(dh->dccph_type),
                                (unsigned long long) lswl, (unsigned long long) seqno,
                                (unsigned long long) dp->dccps_swh,
                                (ackno == DCCP_PKT_WITHOUT_ACK_SEQ) ? "doesn't exist"
                                                                    : "exists",
                                (unsigned long long) lawl, (unsigned long long) ackno,
                                (unsigned long long) dp->dccps_awh);
      
                      dp->dccps_rate_last = now;
      
                      if (dh->dccph_type == DCCP_PKT_RESET)
                              seqno = dp->dccps_gsr;
                      dccp_send_sync(sk, seqno, DCCP_PKT_SYNC);
   57                 return -1;
              }
      
              return 0;
      }
      
      static int __dccp_rcv_established(struct sock *sk, struct sk_buff *skb,
                                        const struct dccp_hdr *dh, const unsigned int len)
      {
              struct dccp_sock *dp = dccp_sk(sk);
      
   45         switch (dccp_hdr(skb)->dccph_type) {
              case DCCP_PKT_DATAACK:
              case DCCP_PKT_DATA:
                      /*
                       * FIXME: schedule DATA_DROPPED (RFC 4340, 11.7.2) if and when
                       * - sk_shutdown == RCV_SHUTDOWN, use Code 1, "Not Listening"
                       * - sk_receive_queue is full, use Code 2, "Receive Buffer"
                       */
    5                 dccp_enqueue_skb(sk, skb);
    5                 return 0;
              case DCCP_PKT_ACK:
                      goto discard;
              case DCCP_PKT_RESET:
                      /*
                       *  Step 9: Process Reset
                       *        If P.type == Reset,
                       *                Tear down connection
                       *                S.state := TIMEWAIT
                       *                Set TIMEWAIT timer
                       *                Drop packet and return
                       */
                      dccp_rcv_reset(sk, skb);
                      return 0;
              case DCCP_PKT_CLOSEREQ:
                      if (dccp_rcv_closereq(sk, skb))
                              return 0;
                      goto discard;
              case DCCP_PKT_CLOSE:
                      if (dccp_rcv_close(sk, skb))
                              return 0;
                      goto discard;
              case DCCP_PKT_REQUEST:
                      /* Step 7
                       *   or (S.is_server and P.type == Response)
                       *   or (S.is_client and P.type == Request)
                       *   or (S.state >= OPEN and P.type == Request
                       *        and P.seqno >= S.OSR)
                       *    or (S.state >= OPEN and P.type == Response
                       *        and P.seqno >= S.OSR)
                       *    or (S.state == RESPOND and P.type == Data),
                       *  Send Sync packet acknowledging P.seqno
                       *  Drop packet and return
                       */
                      if (dp->dccps_role != DCCP_ROLE_LISTEN)
                              goto send_sync;
                      goto check_seq;
              case DCCP_PKT_RESPONSE:
                      if (dp->dccps_role != DCCP_ROLE_CLIENT)
                              goto send_sync;
      check_seq:
                      if (dccp_delta_seqno(dp->dccps_osr,
                                           DCCP_SKB_CB(skb)->dccpd_seq) >= 0) {
      send_sync:
                              dccp_send_sync(sk, DCCP_SKB_CB(skb)->dccpd_seq,
                                             DCCP_PKT_SYNC);
                      }
                      break;
              case DCCP_PKT_SYNC:
                      dccp_send_sync(sk, DCCP_SKB_CB(skb)->dccpd_seq,
                                     DCCP_PKT_SYNCACK);
                      /*
                       * From RFC 4340, sec. 5.7
                       *
                       * As with DCCP-Ack packets, DCCP-Sync and DCCP-SyncAck packets
                       * MAY have non-zero-length application data areas, whose
                       * contents receivers MUST ignore.
                       */
                      goto discard;
              }
      
              DCCP_INC_STATS(DCCP_MIB_INERRS);
      discard:
   45         __kfree_skb(skb);
   45         return 0;
      }
      
   45 int dccp_rcv_established(struct sock *sk, struct sk_buff *skb,
                               const struct dccp_hdr *dh, const unsigned int len)
      {
   45         if (dccp_check_seqno(sk, skb))
                      goto discard;
      
   45         if (dccp_parse_options(sk, NULL, skb))
                      return 1;
      
   45         dccp_handle_ackvec_processing(sk, skb);
              dccp_deliver_input_to_ccids(sk, skb);
      
              return __dccp_rcv_established(sk, skb, dh, len);
      discard:
              __kfree_skb(skb);
              return 0;
      }
      
      EXPORT_SYMBOL_GPL(dccp_rcv_established);
      
      static int dccp_rcv_request_sent_state_process(struct sock *sk,
                                                     struct sk_buff *skb,
                                                     const struct dccp_hdr *dh,
                                                     const unsigned int len)
      {
              /*
               *  Step 4: Prepare sequence numbers in REQUEST
               *     If S.state == REQUEST,
               *          If (P.type == Response or P.type == Reset)
               *                and S.AWL <= P.ackno <= S.AWH,
               *             / * Set sequence number variables corresponding to the
               *                other endpoint, so P will pass the tests in Step 6 * /
               *             Set S.GSR, S.ISR, S.SWL, S.SWH
               *             / * Response processing continues in Step 10; Reset
               *                processing continues in Step 9 * /
              */
   40         if (dh->dccph_type == DCCP_PKT_RESPONSE) {
                      const struct inet_connection_sock *icsk = inet_csk(sk);
                      struct dccp_sock *dp = dccp_sk(sk);
   40                 long tstamp = dccp_timestamp();
      
                      if (!between48(DCCP_SKB_CB(skb)->dccpd_ack_seq,
                                     dp->dccps_awl, dp->dccps_awh)) {
                              dccp_pr_debug("invalid ackno: S.AWL=%llu, "
                                            "P.ackno=%llu, S.AWH=%llu\n",
                                            (unsigned long long)dp->dccps_awl,
                                 (unsigned long long)DCCP_SKB_CB(skb)->dccpd_ack_seq,
                                            (unsigned long long)dp->dccps_awh);
                              goto out_invalid_packet;
                      }
      
                      /*
                       * If option processing (Step 8) failed, return 1 here so that
                       * dccp_v4_do_rcv() sends a Reset. The Reset code depends on
                       * the option type and is set in dccp_parse_options().
                       */
   40                 if (dccp_parse_options(sk, NULL, skb))
                              return 1;
      
                      /* Obtain usec RTT sample from SYN exchange (used by TFRC). */
   40                 if (likely(dp->dccps_options_received.dccpor_timestamp_echo))
                              dp->dccps_syn_rtt = dccp_sample_rtt(sk, 10 * (tstamp -
   40                             dp->dccps_options_received.dccpor_timestamp_echo));
      
                      /* Stop the REQUEST timer */
   40                 inet_csk_clear_xmit_timer(sk, ICSK_TIME_RETRANS);
                      WARN_ON(sk->sk_send_head == NULL);
   40                 kfree_skb(sk->sk_send_head);
                      sk->sk_send_head = NULL;
      
                      /*
                       * Set ISR, GSR from packet. ISS was set in dccp_v{4,6}_connect
                       * and GSS in dccp_transmit_skb(). Setting AWL/AWH and SWL/SWH
                       * is done as part of activating the feature values below, since
                       * these settings depend on the local/remote Sequence Window
                       * features, which were undefined or not confirmed until now.
                       */
                      dp->dccps_gsr = dp->dccps_isr = DCCP_SKB_CB(skb)->dccpd_seq;
      
                      dccp_sync_mss(sk, icsk->icsk_pmtu_cookie);
      
                      /*
                       *    Step 10: Process REQUEST state (second part)
                       *       If S.state == REQUEST,
                       *          / * If we get here, P is a valid Response from the
                       *              server (see Step 4), and we should move to
                       *              PARTOPEN state. PARTOPEN means send an Ack,
                       *              don't send Data packets, retransmit Acks
                       *              periodically, and always include any Init Cookie
                       *              from the Response * /
                       *          S.state := PARTOPEN
                       *          Set PARTOPEN timer
                       *          Continue with S.state == PARTOPEN
                       *          / * Step 12 will send the Ack completing the
                       *              three-way handshake * /
                       */
                      dccp_set_state(sk, DCCP_PARTOPEN);
      
                      /*
                       * If feature negotiation was successful, activate features now;
                       * an activation failure means that this host could not activate
                       * one ore more features (e.g. insufficient memory), which would
                       * leave at least one feature in an undefined state.
                       */
                      if (dccp_feat_activate_values(sk, &dp->dccps_featneg))
                              goto unable_to_proceed;
      
                      /* Make sure socket is routed, for correct metrics. */
   40                 icsk->icsk_af_ops->rebuild_header(sk);
      
                      if (!sock_flag(sk, SOCK_DEAD)) {
   40                         sk->sk_state_change(sk);
                              sk_wake_async(sk, SOCK_WAKE_IO, POLL_OUT);
                      }
      
   40                 if (sk->sk_write_pending || inet_csk_in_pingpong_mode(sk) ||
   40                     icsk->icsk_accept_queue.rskq_defer_accept) {
                              /* Save one ACK. Data will be ready after
                               * several ticks, if write_pending is set.
                               *
                               * It may be deleted, but with this feature tcpdumps
                               * look so _wonderfully_ clever, that I was not able
                               * to stand against the temptation 8)     --ANK
                               */
                              /*
                               * OK, in DCCP we can as well do a similar trick, its
                               * even in the draft, but there is no need for us to
                               * schedule an ack here, as dccp_sendmsg does this for
                               * us, also stated in the draft. -acme
                               */
                              __kfree_skb(skb);
                              return 0;
                      }
   40                 dccp_send_ack(sk);
                      return -1;
              }
      
      out_invalid_packet:
              /* dccp_v4_do_rcv will send a reset */
              DCCP_SKB_CB(skb)->dccpd_reset_code = DCCP_RESET_CODE_PACKET_ERROR;
              return 1;
      
      unable_to_proceed:
              DCCP_SKB_CB(skb)->dccpd_reset_code = DCCP_RESET_CODE_ABORTED;
              /*
               * We mark this socket as no longer usable, so that the loop in
               * dccp_sendmsg() terminates and the application gets notified.
               */
              dccp_set_state(sk, DCCP_CLOSED);
              sk->sk_err = ECOMM;
              return 1;
      }
      
      static int dccp_rcv_respond_partopen_state_process(struct sock *sk,
                                                         struct sk_buff *skb,
                                                         const struct dccp_hdr *dh,
                                                         const unsigned int len)
      {
              struct dccp_sock *dp = dccp_sk(sk);
    9         u32 sample = dp->dccps_options_received.dccpor_timestamp_echo;
              int queued = 0;
      
    9         switch (dh->dccph_type) {
              case DCCP_PKT_RESET:
                      inet_csk_clear_xmit_timer(sk, ICSK_TIME_DACK);
                      break;
              case DCCP_PKT_DATA:
                      if (sk->sk_state == DCCP_RESPOND)
                              break;
                      /* fall through */
              case DCCP_PKT_DATAACK:
              case DCCP_PKT_ACK:
                      /*
                       * FIXME: we should be resetting the PARTOPEN (DELACK) timer
                       * here but only if we haven't used the DELACK timer for
                       * something else, like sending a delayed ack for a TIMESTAMP
                       * echo, etc, for now were not clearing it, sending an extra
                       * ACK when there is nothing else to do in DELACK is not a big
                       * deal after all.
                       */
      
                      /* Stop the PARTOPEN timer */
                      if (sk->sk_state == DCCP_PARTOPEN)
    9                         inet_csk_clear_xmit_timer(sk, ICSK_TIME_DACK);
      
                      /* Obtain usec RTT sample from SYN exchange (used by TFRC). */
    9                 if (likely(sample)) {
                              long delta = dccp_timestamp() - sample;
      
                              dp->dccps_syn_rtt = dccp_sample_rtt(sk, 10 * delta);
                      }
      
    9                 dp->dccps_osr = DCCP_SKB_CB(skb)->dccpd_seq;
                      dccp_set_state(sk, DCCP_OPEN);
      
                      if (dh->dccph_type == DCCP_PKT_DATAACK ||
                          dh->dccph_type == DCCP_PKT_DATA) {
                              __dccp_rcv_established(sk, skb, dh, len);
                              queued = 1; /* packet was queued
                                             (by __dccp_rcv_established) */
                      }
                      break;
              }
      
              return queued;
      }
      
      int dccp_rcv_state_process(struct sock *sk, struct sk_buff *skb,
                                 struct dccp_hdr *dh, unsigned int len)
      {
              struct dccp_sock *dp = dccp_sk(sk);
              struct dccp_skb_cb *dcb = DCCP_SKB_CB(skb);
   58         const int old_state = sk->sk_state;
              bool acceptable;
              int queued = 0;
      
              /*
               *  Step 3: Process LISTEN state
               *
               *     If S.state == LISTEN,
               *         If P.type == Request or P contains a valid Init Cookie option,
               *              (* Must scan the packet's options to check for Init
               *                 Cookies.  Only Init Cookies are processed here,
               *                 however; other options are processed in Step 8.  This
               *                 scan need only be performed if the endpoint uses Init
               *                 Cookies *)
               *              (* Generate a new socket and switch to that socket *)
               *              Set S := new socket for this port pair
               *              S.state = RESPOND
               *              Choose S.ISS (initial seqno) or set from Init Cookies
               *              Initialize S.GAR := S.ISS
               *              Set S.ISR, S.GSR, S.SWL, S.SWH from packet or Init
               *              Cookies Continue with S.state == RESPOND
               *              (* A Response packet will be generated in Step 11 *)
               *         Otherwise,
               *              Generate Reset(No Connection) unless P.type == Reset
               *              Drop packet and return
               */
              if (sk->sk_state == DCCP_LISTEN) {
                      if (dh->dccph_type == DCCP_PKT_REQUEST) {
                              /* It is possible that we process SYN packets from backlog,
                               * so we need to make sure to disable BH and RCU right there.
                               */
                              rcu_read_lock();
                              local_bh_disable();
                              acceptable = inet_csk(sk)->icsk_af_ops->conn_request(sk, skb) >= 0;
                              local_bh_enable();
                              rcu_read_unlock();
                              if (!acceptable)
                                      return 1;
                              consume_skb(skb);
                              return 0;
                      }
                      if (dh->dccph_type == DCCP_PKT_RESET)
                              goto discard;
      
                      /* Caller (dccp_v4_do_rcv) will send Reset */
    1                 dcb->dccpd_reset_code = DCCP_RESET_CODE_NO_CONNECTION;
                      return 1;
   58         } else if (sk->sk_state == DCCP_CLOSED) {
                      dcb->dccpd_reset_code = DCCP_RESET_CODE_NO_CONNECTION;
                      return 1;
              }
      
              /* Step 6: Check sequence numbers (omitted in LISTEN/REQUEST state) */
   57         if (sk->sk_state != DCCP_REQUESTING && dccp_check_seqno(sk, skb))
                      goto discard;
      
              /*
               *   Step 7: Check for unexpected packet types
               *      If (S.is_server and P.type == Response)
               *            or (S.is_client and P.type == Request)
               *            or (S.state == RESPOND and P.type == Data),
               *          Send Sync packet acknowledging P.seqno
               *          Drop packet and return
               */
   57         if ((dp->dccps_role != DCCP_ROLE_CLIENT &&
                   dh->dccph_type == DCCP_PKT_RESPONSE) ||
   57             (dp->dccps_role == DCCP_ROLE_CLIENT &&
                   dh->dccph_type == DCCP_PKT_REQUEST) ||
   57             (sk->sk_state == DCCP_RESPOND && dh->dccph_type == DCCP_PKT_DATA)) {
                      dccp_send_sync(sk, dcb->dccpd_seq, DCCP_PKT_SYNC);
                      goto discard;
              }
      
              /*  Step 8: Process options */
   57         if (dccp_parse_options(sk, NULL, skb))
                      return 1;
      
              /*
               *  Step 9: Process Reset
               *        If P.type == Reset,
               *                Tear down connection
               *                S.state := TIMEWAIT
               *                Set TIMEWAIT timer
               *                Drop packet and return
               */
   53         if (dh->dccph_type == DCCP_PKT_RESET) {
    9                 dccp_rcv_reset(sk, skb);
                      return 0;
   47         } else if (dh->dccph_type == DCCP_PKT_CLOSEREQ) {        /* Step 13 */
                      if (dccp_rcv_closereq(sk, skb))
                              return 0;
                      goto discard;
   47         } else if (dh->dccph_type == DCCP_PKT_CLOSE) {                /* Step 14 */
    2                 if (dccp_rcv_close(sk, skb))
                              return 0;
                      goto discard;
              }
      
   46         switch (sk->sk_state) {
              case DCCP_REQUESTING:
   40                 queued = dccp_rcv_request_sent_state_process(sk, skb, dh, len);
                      if (queued >= 0)
                              return queued;
      
                      __kfree_skb(skb);
                      return 0;
      
              case DCCP_PARTOPEN:
                      /* Step 8: if using Ack Vectors, mark packet acknowledgeable */
    9                 dccp_handle_ackvec_processing(sk, skb);
                      dccp_deliver_input_to_ccids(sk, skb);
                      /* fall through */
              case DCCP_RESPOND:
    9                 queued = dccp_rcv_respond_partopen_state_process(sk, skb,
                                                                       dh, len);
                      break;
              }
      
   10         if (dh->dccph_type == DCCP_PKT_ACK ||
                  dh->dccph_type == DCCP_PKT_DATAACK) {
   10                 switch (old_state) {
                      case DCCP_PARTOPEN:
    9                         sk->sk_state_change(sk);
    2                         sk_wake_async(sk, SOCK_WAKE_IO, POLL_OUT);
                              break;
                      }
              } else if (unlikely(dh->dccph_type == DCCP_PKT_SYNC)) {
                      dccp_send_sync(sk, dcb->dccpd_seq, DCCP_PKT_SYNCACK);
                      goto discard;
              }
      
   10         if (!queued) {
      discard:
   54                 __kfree_skb(skb);
              }
              return 0;
      }
      
      EXPORT_SYMBOL_GPL(dccp_rcv_state_process);
      
      /**
       *  dccp_sample_rtt  -  Validate and finalise computation of RTT sample
       *  @delta:        number of microseconds between packet and acknowledgment
       *
       *  The routine is kept generic to work in different contexts. It should be
       *  called immediately when the ACK used for the RTT sample arrives.
       */
      u32 dccp_sample_rtt(struct sock *sk, long delta)
      {
              /* dccpor_elapsed_time is either zeroed out or set and > 0 */
   40         delta -= dccp_sk(sk)->dccps_options_received.dccpor_elapsed_time * 10;
      
              if (unlikely(delta <= 0)) {
                      DCCP_WARN("unusable RTT sample %ld, using min\n", delta);
                      return DCCP_SANE_RTT_MIN;
              }
   40         if (unlikely(delta > DCCP_SANE_RTT_MAX)) {
                      DCCP_WARN("RTT sample %ld too large, using max\n", delta);
                      return DCCP_SANE_RTT_MAX;
              }
      
   40         return delta;
      }
      // SPDX-License-Identifier: GPL-2.0-only
      /* bpf/cpumap.c
       *
       * Copyright (c) 2017 Jesper Dangaard Brouer, Red Hat Inc.
       */
      
      /* The 'cpumap' is primarily used as a backend map for XDP BPF helper
       * call bpf_redirect_map() and XDP_REDIRECT action, like 'devmap'.
       *
       * Unlike devmap which redirects XDP frames out another NIC device,
       * this map type redirects raw XDP frames to another CPU.  The remote
       * CPU will do SKB-allocation and call the normal network stack.
       *
       * This is a scalability and isolation mechanism, that allow
       * separating the early driver network XDP layer, from the rest of the
       * netstack, and assigning dedicated CPUs for this stage.  This
       * basically allows for 10G wirespeed pre-filtering via bpf.
       */
      #include <linux/bpf.h>
      #include <linux/filter.h>
      #include <linux/ptr_ring.h>
      #include <net/xdp.h>
      
      #include <linux/sched.h>
      #include <linux/workqueue.h>
      #include <linux/kthread.h>
      #include <linux/capability.h>
      #include <trace/events/xdp.h>
      
      #include <linux/netdevice.h>   /* netif_receive_skb_core */
      #include <linux/etherdevice.h> /* eth_type_trans */
      
      /* General idea: XDP packets getting XDP redirected to another CPU,
       * will maximum be stored/queued for one driver ->poll() call.  It is
       * guaranteed that queueing the frame and the flush operation happen on
       * same CPU.  Thus, cpu_map_flush operation can deduct via this_cpu_ptr()
       * which queue in bpf_cpu_map_entry contains packets.
       */
      
      #define CPU_MAP_BULK_SIZE 8  /* 8 == one cacheline on 64-bit archs */
      struct bpf_cpu_map_entry;
      struct bpf_cpu_map;
      
      struct xdp_bulk_queue {
              void *q[CPU_MAP_BULK_SIZE];
              struct list_head flush_node;
              struct bpf_cpu_map_entry *obj;
              unsigned int count;
      };
      
      /* Struct for every remote "destination" CPU in map */
      struct bpf_cpu_map_entry {
              u32 cpu;    /* kthread CPU and map index */
              int map_id; /* Back reference to map */
              u32 qsize;  /* Queue size placeholder for map lookup */
      
              /* XDP can run multiple RX-ring queues, need __percpu enqueue store */
              struct xdp_bulk_queue __percpu *bulkq;
      
              struct bpf_cpu_map *cmap;
      
              /* Queue with potential multi-producers, and single-consumer kthread */
              struct ptr_ring *queue;
              struct task_struct *kthread;
              struct work_struct kthread_stop_wq;
      
              atomic_t refcnt; /* Control when this struct can be free'ed */
              struct rcu_head rcu;
      };
      
      struct bpf_cpu_map {
              struct bpf_map map;
              /* Below members specific for map type */
              struct bpf_cpu_map_entry **cpu_map;
              struct list_head __percpu *flush_list;
      };
      
      static int bq_flush_to_queue(struct xdp_bulk_queue *bq, bool in_napi_ctx);
      
    6 static struct bpf_map *cpu_map_alloc(union bpf_attr *attr)
      {
              struct bpf_cpu_map *cmap;
              int err = -ENOMEM;
              int ret, cpu;
              u64 cost;
      
   10         if (!capable(CAP_SYS_ADMIN))
                      return ERR_PTR(-EPERM);
      
              /* check sanity of attributes */
   10         if (attr->max_entries == 0 || attr->key_size != 4 ||
    8             attr->value_size != 4 || attr->map_flags & ~BPF_F_NUMA_NODE)
                      return ERR_PTR(-EINVAL);
      
    6         cmap = kzalloc(sizeof(*cmap), GFP_USER);
              if (!cmap)
                      return ERR_PTR(-ENOMEM);
      
    6         bpf_map_init_from_attr(&cmap->map, attr);
      
              /* Pre-limit array size based on NR_CPUS, not final CPU check */
              if (cmap->map.max_entries > NR_CPUS) {
                      err = -E2BIG;
                      goto free_cmap;
              }
      
              /* make sure page count doesn't overflow */
    5         cost = (u64) cmap->map.max_entries * sizeof(struct bpf_cpu_map_entry *);
              cost += sizeof(struct list_head) * num_possible_cpus();
      
              /* Notice returns -EPERM on if map size is larger than memlock limit */
              ret = bpf_map_charge_init(&cmap->map.memory, cost);
              if (ret) {
                      err = ret;
                      goto free_cmap;
              }
      
    5         cmap->flush_list = alloc_percpu(struct list_head);
              if (!cmap->flush_list)
                      goto free_charge;
      
    5         for_each_possible_cpu(cpu)
    5                 INIT_LIST_HEAD(per_cpu_ptr(cmap->flush_list, cpu));
      
              /* Alloc array for possible remote "destination" CPUs */
    5         cmap->cpu_map = bpf_map_area_alloc(cmap->map.max_entries *
                                                 sizeof(struct bpf_cpu_map_entry *),
                                                 cmap->map.numa_node);
              if (!cmap->cpu_map)
                      goto free_percpu;
      
              return &cmap->map;
      free_percpu:
              free_percpu(cmap->flush_list);
      free_charge:
              bpf_map_charge_finish(&cmap->map.memory);
      free_cmap:
    1         kfree(cmap);
              return ERR_PTR(err);
      }
      
      static void get_cpu_map_entry(struct bpf_cpu_map_entry *rcpu)
      {
    1         atomic_inc(&rcpu->refcnt);
      }
      
      /* called from workqueue, to workaround syscall using preempt_disable */
      static void cpu_map_kthread_stop(struct work_struct *work)
      {
              struct bpf_cpu_map_entry *rcpu;
      
              rcpu = container_of(work, struct bpf_cpu_map_entry, kthread_stop_wq);
      
              /* Wait for flush in __cpu_map_entry_free(), via full RCU barrier,
               * as it waits until all in-flight call_rcu() callbacks complete.
               */
              rcu_barrier();
      
              /* kthread_stop will wake_up_process and wait for it to complete */
              kthread_stop(rcpu->kthread);
      }
      
      static struct sk_buff *cpu_map_build_skb(struct bpf_cpu_map_entry *rcpu,
                                               struct xdp_frame *xdpf,
                                               struct sk_buff *skb)
      {
              unsigned int hard_start_headroom;
              unsigned int frame_size;
              void *pkt_data_start;
      
              /* Part of headroom was reserved to xdpf */
              hard_start_headroom = sizeof(struct xdp_frame) +  xdpf->headroom;
      
              /* build_skb need to place skb_shared_info after SKB end, and
               * also want to know the memory "truesize".  Thus, need to
               * know the memory frame size backing xdp_buff.
               *
               * XDP was designed to have PAGE_SIZE frames, but this
               * assumption is not longer true with ixgbe and i40e.  It
               * would be preferred to set frame_size to 2048 or 4096
               * depending on the driver.
               *   frame_size = 2048;
               *   frame_len  = frame_size - sizeof(*xdp_frame);
               *
               * Instead, with info avail, skb_shared_info in placed after
               * packet len.  This, unfortunately fakes the truesize.
               * Another disadvantage of this approach, the skb_shared_info
               * is not at a fixed memory location, with mixed length
               * packets, which is bad for cache-line hotness.
               */
              frame_size = SKB_DATA_ALIGN(xdpf->len + hard_start_headroom) +
                      SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
      
              pkt_data_start = xdpf->data - hard_start_headroom;
              skb = build_skb_around(skb, pkt_data_start, frame_size);
              if (unlikely(!skb))
                      return NULL;
      
              skb_reserve(skb, hard_start_headroom);
              __skb_put(skb, xdpf->len);
              if (xdpf->metasize)
                      skb_metadata_set(skb, xdpf->metasize);
      
              /* Essential SKB info: protocol and skb->dev */
              skb->protocol = eth_type_trans(skb, xdpf->dev_rx);
      
              /* Optional SKB info, currently missing:
               * - HW checksum info                (skb->ip_summed)
               * - HW RX hash                        (skb_set_hash)
               * - RX ring dev queue index        (skb_record_rx_queue)
               */
      
              /* Until page_pool get SKB return path, release DMA here */
              xdp_release_frame(xdpf);
      
              /* Allow SKB to reuse area used by xdp_frame */
              xdp_scrub_frame(xdpf);
      
              return skb;
      }
      
      static void __cpu_map_ring_cleanup(struct ptr_ring *ring)
      {
              /* The tear-down procedure should have made sure that queue is
               * empty.  See __cpu_map_entry_replace() and work-queue
               * invoked cpu_map_kthread_stop(). Catch any broken behaviour
               * gracefully and warn once.
               */
              struct xdp_frame *xdpf;
      
              while ((xdpf = ptr_ring_consume(ring)))
                      if (WARN_ON_ONCE(xdpf))
                              xdp_return_frame(xdpf);
      }
      
      static void put_cpu_map_entry(struct bpf_cpu_map_entry *rcpu)
      {
              if (atomic_dec_and_test(&rcpu->refcnt)) {
                      /* The queue should be empty at this point */
                      __cpu_map_ring_cleanup(rcpu->queue);
                      ptr_ring_cleanup(rcpu->queue, NULL);
                      kfree(rcpu->queue);
                      kfree(rcpu);
              }
      }
      
      #define CPUMAP_BATCH 8
      
      static int cpu_map_kthread_run(void *data)
      {
              struct bpf_cpu_map_entry *rcpu = data;
      
              set_current_state(TASK_INTERRUPTIBLE);
      
              /* When kthread gives stop order, then rcpu have been disconnected
               * from map, thus no new packets can enter. Remaining in-flight
               * per CPU stored packets are flushed to this queue.  Wait honoring
               * kthread_stop signal until queue is empty.
               */
              while (!kthread_should_stop() || !__ptr_ring_empty(rcpu->queue)) {
                      unsigned int drops = 0, sched = 0;
                      void *frames[CPUMAP_BATCH];
                      void *skbs[CPUMAP_BATCH];
                      gfp_t gfp = __GFP_ZERO | GFP_ATOMIC;
                      int i, n, m;
      
                      /* Release CPU reschedule checks */
                      if (__ptr_ring_empty(rcpu->queue)) {
                              set_current_state(TASK_INTERRUPTIBLE);
                              /* Recheck to avoid lost wake-up */
                              if (__ptr_ring_empty(rcpu->queue)) {
                                      schedule();
                                      sched = 1;
                              } else {
                                      __set_current_state(TASK_RUNNING);
                              }
                      } else {
                              sched = cond_resched();
                      }
      
                      /*
                       * The bpf_cpu_map_entry is single consumer, with this
                       * kthread CPU pinned. Lockless access to ptr_ring
                       * consume side valid as no-resize allowed of queue.
                       */
                      n = ptr_ring_consume_batched(rcpu->queue, frames, CPUMAP_BATCH);
      
                      for (i = 0; i < n; i++) {
                              void *f = frames[i];
                              struct page *page = virt_to_page(f);
      
                              /* Bring struct page memory area to curr CPU. Read by
                               * build_skb_around via page_is_pfmemalloc(), and when
                               * freed written by page_frag_free call.
                               */
                              prefetchw(page);
                      }
      
                      m = kmem_cache_alloc_bulk(skbuff_head_cache, gfp, n, skbs);
                      if (unlikely(m == 0)) {
                              for (i = 0; i < n; i++)
                                      skbs[i] = NULL; /* effect: xdp_return_frame */
                              drops = n;
                      }
      
                      local_bh_disable();
                      for (i = 0; i < n; i++) {
                              struct xdp_frame *xdpf = frames[i];
                              struct sk_buff *skb = skbs[i];
                              int ret;
      
                              skb = cpu_map_build_skb(rcpu, xdpf, skb);
                              if (!skb) {
                                      xdp_return_frame(xdpf);
                                      continue;
                              }
      
                              /* Inject into network stack */
                              ret = netif_receive_skb_core(skb);
                              if (ret == NET_RX_DROP)
                                      drops++;
                      }
                      /* Feedback loop via tracepoint */
                      trace_xdp_cpumap_kthread(rcpu->map_id, n, drops, sched);
      
                      local_bh_enable(); /* resched point, may call do_softirq() */
              }
              __set_current_state(TASK_RUNNING);
      
              put_cpu_map_entry(rcpu);
              return 0;
      }
      
      static struct bpf_cpu_map_entry *__cpu_map_entry_alloc(u32 qsize, u32 cpu,
                                                             int map_id)
      {
              gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
              struct bpf_cpu_map_entry *rcpu;
              struct xdp_bulk_queue *bq;
              int numa, err, i;
      
              /* Have map->numa_node, but choose node of redirect target CPU */
              numa = cpu_to_node(cpu);
      
              rcpu = kzalloc_node(sizeof(*rcpu), gfp, numa);
              if (!rcpu)
                      return NULL;
      
              /* Alloc percpu bulkq */
    2         rcpu->bulkq = __alloc_percpu_gfp(sizeof(*rcpu->bulkq),
                                               sizeof(void *), gfp);
              if (!rcpu->bulkq)
                      goto free_rcu;
      
    2         for_each_possible_cpu(i) {
    2                 bq = per_cpu_ptr(rcpu->bulkq, i);
                      bq->obj = rcpu;
              }
      
              /* Alloc queue */
    2         rcpu->queue = kzalloc_node(sizeof(*rcpu->queue), gfp, numa);
              if (!rcpu->queue)
                      goto free_bulkq;
      
    2         err = ptr_ring_init(rcpu->queue, qsize, gfp);
              if (err)
                      goto free_queue;
      
              rcpu->cpu    = cpu;
              rcpu->map_id = map_id;
              rcpu->qsize  = qsize;
      
              /* Setup kthread */
              rcpu->kthread = kthread_create_on_node(cpu_map_kthread_run, rcpu, numa,
                                                     "cpumap/%d/map:%d", cpu, map_id);
              if (IS_ERR(rcpu->kthread))
                      goto free_ptr_ring;
      
    1         get_cpu_map_entry(rcpu); /* 1-refcnt for being in cmap->cpu_map[] */
              get_cpu_map_entry(rcpu); /* 1-refcnt for kthread */
      
              /* Make sure kthread runs on a single CPU */
              kthread_bind(rcpu->kthread, cpu);
              wake_up_process(rcpu->kthread);
      
              return rcpu;
      
      free_ptr_ring:
              ptr_ring_cleanup(rcpu->queue, NULL);
      free_queue:
              kfree(rcpu->queue);
      free_bulkq:
              free_percpu(rcpu->bulkq);
      free_rcu:
              kfree(rcpu);
              return NULL;
      }
      
      static void __cpu_map_entry_free(struct rcu_head *rcu)
      {
              struct bpf_cpu_map_entry *rcpu;
              int cpu;
      
              /* This cpu_map_entry have been disconnected from map and one
               * RCU graze-period have elapsed.  Thus, XDP cannot queue any
               * new packets and cannot change/set flush_needed that can
               * find this entry.
               */
              rcpu = container_of(rcu, struct bpf_cpu_map_entry, rcu);
      
              /* Flush remaining packets in percpu bulkq */
              for_each_online_cpu(cpu) {
                      struct xdp_bulk_queue *bq = per_cpu_ptr(rcpu->bulkq, cpu);
      
                      /* No concurrent bq_enqueue can run at this point */
                      bq_flush_to_queue(bq, false);
              }
              free_percpu(rcpu->bulkq);
              /* Cannot kthread_stop() here, last put free rcpu resources */
              put_cpu_map_entry(rcpu);
      }
      
      /* After xchg pointer to bpf_cpu_map_entry, use the call_rcu() to
       * ensure any driver rcu critical sections have completed, but this
       * does not guarantee a flush has happened yet. Because driver side
       * rcu_read_lock/unlock only protects the running XDP program.  The
       * atomic xchg and NULL-ptr check in __cpu_map_flush() makes sure a
       * pending flush op doesn't fail.
       *
       * The bpf_cpu_map_entry is still used by the kthread, and there can
       * still be pending packets (in queue and percpu bulkq).  A refcnt
       * makes sure to last user (kthread_stop vs. call_rcu) free memory
       * resources.
       *
       * The rcu callback __cpu_map_entry_free flush remaining packets in
       * percpu bulkq to queue.  Due to caller map_delete_elem() disable
       * preemption, cannot call kthread_stop() to make sure queue is empty.
       * Instead a work_queue is started for stopping kthread,
       * cpu_map_kthread_stop, which waits for an RCU graze period before
       * stopping kthread, emptying the queue.
       */
      static void __cpu_map_entry_replace(struct bpf_cpu_map *cmap,
                                          u32 key_cpu, struct bpf_cpu_map_entry *rcpu)
      {
              struct bpf_cpu_map_entry *old_rcpu;
      
    3         old_rcpu = xchg(&cmap->cpu_map[key_cpu], rcpu);
              if (old_rcpu) {
    2                 call_rcu(&old_rcpu->rcu, __cpu_map_entry_free);
                      INIT_WORK(&old_rcpu->kthread_stop_wq, cpu_map_kthread_stop);
                      schedule_work(&old_rcpu->kthread_stop_wq);
              }
    2 }
      
      static int cpu_map_delete_elem(struct bpf_map *map, void *key)
      {
              struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
    3         u32 key_cpu = *(u32 *)key;
      
              if (key_cpu >= map->max_entries)
                      return -EINVAL;
      
              /* notice caller map_delete_elem() use preempt_disable() */
    2         __cpu_map_entry_replace(cmap, key_cpu, NULL);
    2         return 0;
      }
      
      static int cpu_map_update_elem(struct bpf_map *map, void *key, void *value,
                                     u64 map_flags)
      {
              struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
              struct bpf_cpu_map_entry *rcpu;
      
              /* Array index key correspond to CPU number */
    8         u32 key_cpu = *(u32 *)key;
              /* Value is the queue size */
    5         u32 qsize = *(u32 *)value;
      
              if (unlikely(map_flags > BPF_EXIST))
                      return -EINVAL;
    7         if (unlikely(key_cpu >= cmap->map.max_entries))
                      return -E2BIG;
    6         if (unlikely(map_flags == BPF_NOEXIST))
                      return -EEXIST;
              if (unlikely(qsize > 16384)) /* sanity limit on qsize */
                      return -EOVERFLOW;
      
              /* Make sure CPU is a valid possible cpu */
    4         if (!cpu_possible(key_cpu))
                      return -ENODEV;
      
    3         if (qsize == 0) {
                      rcpu = NULL; /* Same as deleting */
              } else {
                      /* Updating qsize cause re-allocation of bpf_cpu_map_entry */
    2                 rcpu = __cpu_map_entry_alloc(qsize, key_cpu, map->id);
                      if (!rcpu)
                              return -ENOMEM;
                      rcpu->cmap = cmap;
              }
    2         rcu_read_lock();
    2         __cpu_map_entry_replace(cmap, key_cpu, rcpu);
    7         rcu_read_unlock();
              return 0;
      }
      
      static void cpu_map_free(struct bpf_map *map)
      {
              struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
              int cpu;
              u32 i;
      
              /* At this point bpf_prog->aux->refcnt == 0 and this map->refcnt == 0,
               * so the bpf programs (can be more than one that used this map) were
               * disconnected from events. Wait for outstanding critical sections in
               * these programs to complete. The rcu critical section only guarantees
               * no further "XDP/bpf-side" reads against bpf_cpu_map->cpu_map.
               * It does __not__ ensure pending flush operations (if any) are
               * complete.
               */
      
    1         bpf_clear_redirect_map(map);
              synchronize_rcu();
      
              /* To ensure all pending flush operations have completed wait for flush
               * list be empty on _all_ cpus. Because the above synchronize_rcu()
               * ensures the map is disconnected from the program we can assume no new
               * items will be added to the list.
               */
    1         for_each_online_cpu(cpu) {
    1                 struct list_head *flush_list = per_cpu_ptr(cmap->flush_list, cpu);
      
                      while (!list_empty(flush_list))
                              cond_resched();
              }
      
              /* For cpu_map the remote CPUs can still be using the entries
               * (struct bpf_cpu_map_entry).
               */
    1         for (i = 0; i < cmap->map.max_entries; i++) {
                      struct bpf_cpu_map_entry *rcpu;
      
    1                 rcpu = READ_ONCE(cmap->cpu_map[i]);
                      if (!rcpu)
                              continue;
      
                      /* bq flush and cleanup happens after RCU graze-period */
                      __cpu_map_entry_replace(cmap, i, NULL); /* call_rcu */
              }
    1         free_percpu(cmap->flush_list);
              bpf_map_area_free(cmap->cpu_map);
              kfree(cmap);
      }
      
      struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key)
      {
              struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
              struct bpf_cpu_map_entry *rcpu;
      
              if (key >= map->max_entries)
                      return NULL;
      
    1         rcpu = READ_ONCE(cmap->cpu_map[key]);
              return rcpu;
      }
      
      static void *cpu_map_lookup_elem(struct bpf_map *map, void *key)
      {
              struct bpf_cpu_map_entry *rcpu =
    2                 __cpu_map_lookup_elem(map, *(u32 *)key);
      
    2         return rcpu ? &rcpu->qsize : NULL;
      }
      
      static int cpu_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
      {
              struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
    5         u32 index = key ? *(u32 *)key : U32_MAX;
              u32 *next = next_key;
      
              if (index >= cmap->map.max_entries) {
    3                 *next = 0;
    3                 return 0;
              }
      
    2         if (index == cmap->map.max_entries - 1)
                      return -ENOENT;
    1         *next = index + 1;
    2         return 0;
      }
      
      const struct bpf_map_ops cpu_map_ops = {
              .map_alloc                = cpu_map_alloc,
              .map_free                = cpu_map_free,
              .map_delete_elem        = cpu_map_delete_elem,
              .map_update_elem        = cpu_map_update_elem,
              .map_lookup_elem        = cpu_map_lookup_elem,
              .map_get_next_key        = cpu_map_get_next_key,
              .map_check_btf                = map_check_no_btf,
      };
      
      static int bq_flush_to_queue(struct xdp_bulk_queue *bq, bool in_napi_ctx)
      {
              struct bpf_cpu_map_entry *rcpu = bq->obj;
              unsigned int processed = 0, drops = 0;
              const int to_cpu = rcpu->cpu;
              struct ptr_ring *q;
              int i;
      
              if (unlikely(!bq->count))
                      return 0;
      
              q = rcpu->queue;
              spin_lock(&q->producer_lock);
      
              for (i = 0; i < bq->count; i++) {
                      struct xdp_frame *xdpf = bq->q[i];
                      int err;
      
                      err = __ptr_ring_produce(q, xdpf);
                      if (err) {
                              drops++;
                              if (likely(in_napi_ctx))
                                      xdp_return_frame_rx_napi(xdpf);
                              else
                                      xdp_return_frame(xdpf);
                      }
                      processed++;
              }
              bq->count = 0;
              spin_unlock(&q->producer_lock);
      
              __list_del_clearprev(&bq->flush_node);
      
              /* Feedback loop via tracepoints */
              trace_xdp_cpumap_enqueue(rcpu->map_id, processed, drops, to_cpu);
              return 0;
      }
      
      /* Runs under RCU-read-side, plus in softirq under NAPI protection.
       * Thus, safe percpu variable access.
       */
      static int bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf)
      {
              struct list_head *flush_list = this_cpu_ptr(rcpu->cmap->flush_list);
              struct xdp_bulk_queue *bq = this_cpu_ptr(rcpu->bulkq);
      
              if (unlikely(bq->count == CPU_MAP_BULK_SIZE))
                      bq_flush_to_queue(bq, true);
      
              /* Notice, xdp_buff/page MUST be queued here, long enough for
               * driver to code invoking us to finished, due to driver
               * (e.g. ixgbe) recycle tricks based on page-refcnt.
               *
               * Thus, incoming xdp_frame is always queued here (else we race
               * with another CPU on page-refcnt and remaining driver code).
               * Queue time is very short, as driver will invoke flush
               * operation, when completing napi->poll call.
               */
              bq->q[bq->count++] = xdpf;
      
              if (!bq->flush_node.prev)
                      list_add(&bq->flush_node, flush_list);
      
              return 0;
      }
      
      int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp,
                          struct net_device *dev_rx)
      {
              struct xdp_frame *xdpf;
      
              xdpf = convert_to_xdp_frame(xdp);
              if (unlikely(!xdpf))
                      return -EOVERFLOW;
      
              /* Info needed when constructing SKB on remote CPU */
              xdpf->dev_rx = dev_rx;
      
              bq_enqueue(rcpu, xdpf);
              return 0;
      }
      
      void __cpu_map_flush(struct bpf_map *map)
      {
              struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
              struct list_head *flush_list = this_cpu_ptr(cmap->flush_list);
              struct xdp_bulk_queue *bq, *tmp;
      
              list_for_each_entry_safe(bq, tmp, flush_list, flush_node) {
                      bq_flush_to_queue(bq, true);
      
                      /* If already running, costs spin_lock_irqsave + smb_mb */
                      wake_up_process(bq->obj->kthread);
              }
      }
      // SPDX-License-Identifier: GPL-2.0-or-later
      /*
       * Asynchronous Cryptographic Hash operations.
       *
       * This is the asynchronous version of hash.c with notification of
       * completion via a callback.
       *
       * Copyright (c) 2008 Loc Ho <lho@amcc.com>
       */
      
      #include <crypto/internal/hash.h>
      #include <crypto/scatterwalk.h>
      #include <linux/bug.h>
      #include <linux/err.h>
      #include <linux/kernel.h>
      #include <linux/module.h>
      #include <linux/sched.h>
      #include <linux/slab.h>
      #include <linux/seq_file.h>
      #include <linux/cryptouser.h>
      #include <linux/compiler.h>
      #include <net/netlink.h>
      
      #include "internal.h"
      
      struct ahash_request_priv {
              crypto_completion_t complete;
              void *data;
              u8 *result;
              u32 flags;
              void *ubuf[] CRYPTO_MINALIGN_ATTR;
      };
      
      static inline struct ahash_alg *crypto_ahash_alg(struct crypto_ahash *hash)
      {
   97         return container_of(crypto_hash_alg_common(hash), struct ahash_alg,
                                  halg);
      }
      
      static int hash_walk_next(struct crypto_hash_walk *walk)
      {
  234         unsigned int alignmask = walk->alignmask;
              unsigned int offset = walk->offset;
              unsigned int nbytes = min(walk->entrylen,
                                        ((unsigned int)(PAGE_SIZE)) - offset);
      
              if (walk->flags & CRYPTO_ALG_ASYNC)
                      walk->data = kmap(walk->pg);
              else
  234                 walk->data = kmap_atomic(walk->pg);
  234         walk->data += offset;
      
              if (offset & alignmask) {
    1                 unsigned int unaligned = alignmask + 1 - (offset & alignmask);
      
                      if (nbytes > unaligned)
                              nbytes = unaligned;
              }
      
  234         walk->entrylen -= nbytes;
              return nbytes;
      }
      
      static int hash_walk_new_entry(struct crypto_hash_walk *walk)
      {
              struct scatterlist *sg;
      
  234         sg = walk->sg;
              walk->offset = sg->offset;
  234         walk->pg = sg_page(walk->sg) + (walk->offset >> PAGE_SHIFT);
              walk->offset = offset_in_page(walk->offset);
              walk->entrylen = sg->length;
      
  234         if (walk->entrylen > walk->total)
    3                 walk->entrylen = walk->total;
  234         walk->total -= walk->entrylen;
      
              return hash_walk_next(walk);
      }
      
      int crypto_hash_walk_done(struct crypto_hash_walk *walk, int err)
  232 {
  232         unsigned int alignmask = walk->alignmask;
      
              walk->data -= walk->offset;
      
   16         if (walk->entrylen && (walk->offset & alignmask) && !err) {
                      unsigned int nbytes;
      
                      walk->offset = ALIGN(walk->offset, alignmask + 1);
                      nbytes = min(walk->entrylen,
                                   (unsigned int)(PAGE_SIZE - walk->offset));
                      if (nbytes) {
                              walk->entrylen -= nbytes;
                              walk->data += walk->offset;
                              return nbytes;
                      }
              }
      
  232         if (walk->flags & CRYPTO_ALG_ASYNC)
                      kunmap(walk->pg);
              else {
  232                 kunmap_atomic(walk->data);
                      /*
                       * The may sleep test only makes sense for sync users.
                       * Async users don't need to sleep here anyway.
                       */
  232                 crypto_yield(walk->flags);
              }
      
  232         if (err)
                      return err;
      
  232         if (walk->entrylen) {
   16                 walk->offset = 0;
                      walk->pg++;
                      return hash_walk_next(walk);
              }
      
  232         if (!walk->total)
                      return 0;
      
   58         walk->sg = sg_next(walk->sg);
      
              return hash_walk_new_entry(walk);
      }
      EXPORT_SYMBOL_GPL(crypto_hash_walk_done);
      
  234 int crypto_hash_walk_first(struct ahash_request *req,
                                 struct crypto_hash_walk *walk)
      {
  234         walk->total = req->nbytes;
      
              if (!walk->total) {
   27                 walk->entrylen = 0;
   27                 return 0;
              }
      
  234         walk->alignmask = crypto_ahash_alignmask(crypto_ahash_reqtfm(req));
              walk->sg = req->src;
              walk->flags = req->base.flags & CRYPTO_TFM_REQ_MASK;
      
  234         return hash_walk_new_entry(walk);
      }
      EXPORT_SYMBOL_GPL(crypto_hash_walk_first);
      
      int crypto_ahash_walk_first(struct ahash_request *req,
                                  struct crypto_hash_walk *walk)
      {
              walk->total = req->nbytes;
      
              if (!walk->total) {
                      walk->entrylen = 0;
                      return 0;
              }
      
              walk->alignmask = crypto_ahash_alignmask(crypto_ahash_reqtfm(req));
              walk->sg = req->src;
              walk->flags = req->base.flags & CRYPTO_TFM_REQ_MASK;
              walk->flags |= CRYPTO_ALG_ASYNC;
      
              BUILD_BUG_ON(CRYPTO_TFM_REQ_MASK & CRYPTO_ALG_ASYNC);
      
              return hash_walk_new_entry(walk);
      }
      EXPORT_SYMBOL_GPL(crypto_ahash_walk_first);
      
      static int ahash_setkey_unaligned(struct crypto_ahash *tfm, const u8 *key,
                                      unsigned int keylen)
      {
              unsigned long alignmask = crypto_ahash_alignmask(tfm);
              int ret;
              u8 *buffer, *alignbuffer;
              unsigned long absize;
      
              absize = keylen + alignmask;
              buffer = kmalloc(absize, GFP_KERNEL);
              if (!buffer)
                      return -ENOMEM;
      
              alignbuffer = (u8 *)ALIGN((unsigned long)buffer, alignmask + 1);
              memcpy(alignbuffer, key, keylen);
              ret = tfm->setkey(tfm, alignbuffer, keylen);
              kzfree(buffer);
              return ret;
      }
      
      static int ahash_nosetkey(struct crypto_ahash *tfm, const u8 *key,
                                unsigned int keylen)
      {
    1         return -ENOSYS;
      }
      
      static void ahash_set_needkey(struct crypto_ahash *tfm)
      {
   14         const struct hash_alg_common *alg = crypto_hash_alg_common(tfm);
      
   15         if (tfm->setkey != ahash_nosetkey &&
    3             !(alg->base.cra_flags & CRYPTO_ALG_OPTIONAL_KEY))
   14                 crypto_ahash_set_flags(tfm, CRYPTO_TFM_NEED_KEY);
      }
      
      int crypto_ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
                              unsigned int keylen)
      {
   89         unsigned long alignmask = crypto_ahash_alignmask(tfm);
              int err;
      
              if ((unsigned long)key & alignmask)
                      err = ahash_setkey_unaligned(tfm, key, keylen);
              else
   89                 err = tfm->setkey(tfm, key, keylen);
      
   86         if (unlikely(err)) {
   15                 ahash_set_needkey(tfm);
                      return err;
              }
      
   71         crypto_ahash_clear_flags(tfm, CRYPTO_TFM_NEED_KEY);
   86         return 0;
      }
      EXPORT_SYMBOL_GPL(crypto_ahash_setkey);
      
      static inline unsigned int ahash_align_buffer_size(unsigned len,
                                                         unsigned long mask)
      {
    1         return len + (mask & ~(crypto_tfm_ctx_alignment() - 1));
      }
      
      static int ahash_save_req(struct ahash_request *req, crypto_completion_t cplt)
      {
    1         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
              unsigned long alignmask = crypto_ahash_alignmask(tfm);
              unsigned int ds = crypto_ahash_digestsize(tfm);
              struct ahash_request_priv *priv;
      
    1         priv = kmalloc(sizeof(*priv) + ahash_align_buffer_size(ds, alignmask),
                             (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
                             GFP_KERNEL : GFP_ATOMIC);
              if (!priv)
                      return -ENOMEM;
      
              /*
               * WARNING: Voodoo programming below!
               *
               * The code below is obscure and hard to understand, thus explanation
               * is necessary. See include/crypto/hash.h and include/linux/crypto.h
               * to understand the layout of structures used here!
               *
               * The code here will replace portions of the ORIGINAL request with
               * pointers to new code and buffers so the hashing operation can store
               * the result in aligned buffer. We will call the modified request
               * an ADJUSTED request.
               *
               * The newly mangled request will look as such:
               *
               * req {
               *   .result        = ADJUSTED[new aligned buffer]
               *   .base.complete = ADJUSTED[pointer to completion function]
               *   .base.data     = ADJUSTED[*req (pointer to self)]
               *   .priv          = ADJUSTED[new priv] {
               *           .result   = ORIGINAL(result)
               *           .complete = ORIGINAL(base.complete)
               *           .data     = ORIGINAL(base.data)
               *   }
               */
      
    1         priv->result = req->result;
              priv->complete = req->base.complete;
              priv->data = req->base.data;
              priv->flags = req->base.flags;
      
              /*
               * WARNING: We do not backup req->priv here! The req->priv
               *          is for internal use of the Crypto API and the
               *          user must _NOT_ _EVER_ depend on it's content!
               */
      
              req->result = PTR_ALIGN((u8 *)priv->ubuf, alignmask + 1);
              req->base.complete = cplt;
              req->base.data = req;
              req->priv = priv;
      
    1         return 0;
      }
      
      static void ahash_restore_req(struct ahash_request *req, int err)
      {
    1         struct ahash_request_priv *priv = req->priv;
      
              if (!err)
                      memcpy(priv->result, req->result,
    1                        crypto_ahash_digestsize(crypto_ahash_reqtfm(req)));
      
              /* Restore the original crypto request. */
    1         req->result = priv->result;
      
              ahash_request_set_callback(req, priv->flags,
                                         priv->complete, priv->data);
              req->priv = NULL;
      
              /* Free the req->priv.priv from the ADJUSTED request. */
              kzfree(priv);
      }
      
      static void ahash_notify_einprogress(struct ahash_request *req)
      {
              struct ahash_request_priv *priv = req->priv;
              struct crypto_async_request oreq;
      
              oreq.data = priv->data;
      
              priv->complete(&oreq, -EINPROGRESS);
      }
      
      static void ahash_op_unaligned_done(struct crypto_async_request *req, int err)
      {
              struct ahash_request *areq = req->data;
      
              if (err == -EINPROGRESS) {
                      ahash_notify_einprogress(areq);
                      return;
              }
      
              /*
               * Restore the original request, see ahash_op_unaligned() for what
               * goes where.
               *
               * The "struct ahash_request *req" here is in fact the "req.base"
               * from the ADJUSTED request from ahash_op_unaligned(), thus as it
               * is a pointer to self, it is also the ADJUSTED "req" .
               */
      
              /* First copy req->result into req->priv.result */
              ahash_restore_req(areq, err);
      
              /* Complete the ORIGINAL request. */
              areq->base.complete(&areq->base, err);
      }
      
      static int ahash_op_unaligned(struct ahash_request *req,
                                    int (*op)(struct ahash_request *))
      {
              int err;
      
              err = ahash_save_req(req, ahash_op_unaligned_done);
              if (err)
                      return err;
      
              err = op(req);
              if (err == -EINPROGRESS || err == -EBUSY)
                      return err;
      
              ahash_restore_req(req, err);
      
              return err;
      }
      
      static int crypto_ahash_op(struct ahash_request *req,
                                 int (*op)(struct ahash_request *))
      {
  253         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
              unsigned long alignmask = crypto_ahash_alignmask(tfm);
      
              if ((unsigned long)req->result & alignmask)
                      return ahash_op_unaligned(req, op);
      
  253         return op(req);
      }
      
      int crypto_ahash_final(struct ahash_request *req)
      {
  140         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
              struct crypto_alg *alg = tfm->base.__crt_alg;
              unsigned int nbytes = req->nbytes;
              int ret;
      
              crypto_stats_get(alg);
              ret = crypto_ahash_op(req, crypto_ahash_reqtfm(req)->final);
              crypto_stats_ahash_final(nbytes, ret, alg);
              return ret;
      }
      EXPORT_SYMBOL_GPL(crypto_ahash_final);
      
      int crypto_ahash_finup(struct ahash_request *req)
      {
   84         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
              struct crypto_alg *alg = tfm->base.__crt_alg;
              unsigned int nbytes = req->nbytes;
              int ret;
      
              crypto_stats_get(alg);
              ret = crypto_ahash_op(req, crypto_ahash_reqtfm(req)->finup);
              crypto_stats_ahash_final(nbytes, ret, alg);
              return ret;
      }
      EXPORT_SYMBOL_GPL(crypto_ahash_finup);
      
      int crypto_ahash_digest(struct ahash_request *req)
      {
   32         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
              struct crypto_alg *alg = tfm->base.__crt_alg;
              unsigned int nbytes = req->nbytes;
              int ret;
      
              crypto_stats_get(alg);
              if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
                      ret = -ENOKEY;
              else
   32                 ret = crypto_ahash_op(req, tfm->digest);
              crypto_stats_ahash_final(nbytes, ret, alg);
   32         return ret;
      }
      EXPORT_SYMBOL_GPL(crypto_ahash_digest);
      
      static void ahash_def_finup_done2(struct crypto_async_request *req, int err)
      {
              struct ahash_request *areq = req->data;
      
              if (err == -EINPROGRESS)
                      return;
      
              ahash_restore_req(areq, err);
      
              areq->base.complete(&areq->base, err);
      }
      
      static int ahash_def_finup_finish1(struct ahash_request *req, int err)
      {
    1         if (err)
                      goto out;
      
    1         req->base.complete = ahash_def_finup_done2;
      
              err = crypto_ahash_reqtfm(req)->final(req);
    1         if (err == -EINPROGRESS || err == -EBUSY)
                      return err;
      
      out:
    1         ahash_restore_req(req, err);
    1         return err;
      }
      
      static void ahash_def_finup_done1(struct crypto_async_request *req, int err)
      {
              struct ahash_request *areq = req->data;
      
              if (err == -EINPROGRESS) {
                      ahash_notify_einprogress(areq);
                      return;
              }
      
              areq->base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
      
              err = ahash_def_finup_finish1(areq, err);
              if (areq->priv)
                      return;
      
              areq->base.complete(&areq->base, err);
      }
      
      static int ahash_def_finup(struct ahash_request *req)
      {
    1         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
              int err;
      
              err = ahash_save_req(req, ahash_def_finup_done1);
              if (err)
                      return err;
      
    1         err = tfm->update(req);
    1         if (err == -EINPROGRESS || err == -EBUSY)
                      return err;
      
    1         return ahash_def_finup_finish1(req, err);
      }
      
      static int crypto_ahash_init_tfm(struct crypto_tfm *tfm)
      {
              struct crypto_ahash *hash = __crypto_ahash_cast(tfm);
   97         struct ahash_alg *alg = crypto_ahash_alg(hash);
      
              hash->setkey = ahash_nosetkey;
      
              if (tfm->__crt_alg->cra_type != &crypto_ahash_type)
   93                 return crypto_init_shash_ops_async(tfm);
      
    4         hash->init = alg->init;
              hash->update = alg->update;
              hash->final = alg->final;
    4         hash->finup = alg->finup ?: ahash_def_finup;
              hash->digest = alg->digest;
              hash->export = alg->export;
              hash->import = alg->import;
      
              if (alg->setkey) {
    3                 hash->setkey = alg->setkey;
    4                 ahash_set_needkey(hash);
              }
      
              return 0;
      }
      
      static unsigned int crypto_ahash_extsize(struct crypto_alg *alg)
      {
  101         if (alg->cra_type != &crypto_ahash_type)
                      return sizeof(struct crypto_shash *);
      
  101         return crypto_alg_extsize(alg);
      }
      
      #ifdef CONFIG_NET
      static int crypto_ahash_report(struct sk_buff *skb, struct crypto_alg *alg)
      {
              struct crypto_report_hash rhash;
      
    3         memset(&rhash, 0, sizeof(rhash));
      
              strscpy(rhash.type, "ahash", sizeof(rhash.type));
      
              rhash.blocksize = alg->cra_blocksize;
              rhash.digestsize = __crypto_hash_alg_common(alg)->digestsize;
      
              return nla_put(skb, CRYPTOCFGA_REPORT_HASH, sizeof(rhash), &rhash);
      }
      #else
      static int crypto_ahash_report(struct sk_buff *skb, struct crypto_alg *alg)
      {
              return -ENOSYS;
      }
      #endif
      
      static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg)
              __maybe_unused;
      static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg)
      {
              seq_printf(m, "type         : ahash\n");
              seq_printf(m, "async        : %s\n", alg->cra_flags & CRYPTO_ALG_ASYNC ?
                                                   "yes" : "no");
              seq_printf(m, "blocksize    : %u\n", alg->cra_blocksize);
              seq_printf(m, "digestsize   : %u\n",
                         __crypto_hash_alg_common(alg)->digestsize);
      }
      
      const struct crypto_type crypto_ahash_type = {
              .extsize = crypto_ahash_extsize,
              .init_tfm = crypto_ahash_init_tfm,
      #ifdef CONFIG_PROC_FS
              .show = crypto_ahash_show,
      #endif
              .report = crypto_ahash_report,
              .maskclear = ~CRYPTO_ALG_TYPE_MASK,
              .maskset = CRYPTO_ALG_TYPE_AHASH_MASK,
              .type = CRYPTO_ALG_TYPE_AHASH,
              .tfmsize = offsetof(struct crypto_ahash, base),
      };
      EXPORT_SYMBOL_GPL(crypto_ahash_type);
      
      struct crypto_ahash *crypto_alloc_ahash(const char *alg_name, u32 type,
                                              u32 mask)
      {
   45         return crypto_alloc_tfm(alg_name, &crypto_ahash_type, type, mask);
      }
      EXPORT_SYMBOL_GPL(crypto_alloc_ahash);
      
      int crypto_has_ahash(const char *alg_name, u32 type, u32 mask)
      {
   11         return crypto_type_has_alg(alg_name, &crypto_ahash_type, type, mask);
      }
      EXPORT_SYMBOL_GPL(crypto_has_ahash);
      
      static int ahash_prepare_alg(struct ahash_alg *alg)
      {
              struct crypto_alg *base = &alg->halg.base;
      
              if (alg->halg.digestsize > HASH_MAX_DIGESTSIZE ||
                  alg->halg.statesize > HASH_MAX_STATESIZE ||
                  alg->halg.statesize == 0)
                      return -EINVAL;
      
              base->cra_type = &crypto_ahash_type;
              base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK;
              base->cra_flags |= CRYPTO_ALG_TYPE_AHASH;
      
              return 0;
      }
      
      int crypto_register_ahash(struct ahash_alg *alg)
      {
              struct crypto_alg *base = &alg->halg.base;
              int err;
      
              err = ahash_prepare_alg(alg);
              if (err)
                      return err;
      
              return crypto_register_alg(base);
      }
      EXPORT_SYMBOL_GPL(crypto_register_ahash);
      
      int crypto_unregister_ahash(struct ahash_alg *alg)
      {
              return crypto_unregister_alg(&alg->halg.base);
      }
      EXPORT_SYMBOL_GPL(crypto_unregister_ahash);
      
      int crypto_register_ahashes(struct ahash_alg *algs, int count)
      {
              int i, ret;
      
              for (i = 0; i < count; i++) {
                      ret = crypto_register_ahash(&algs[i]);
                      if (ret)
                              goto err;
              }
      
              return 0;
      
      err:
              for (--i; i >= 0; --i)
                      crypto_unregister_ahash(&algs[i]);
      
              return ret;
      }
      EXPORT_SYMBOL_GPL(crypto_register_ahashes);
      
      void cr